@utaba/deep-memory 0.14.0 → 0.16.0

package/dist/index.js

@@ -165,6 +165,18 @@ var OperationCancelledError = class extends DeepMemoryError {
     this.reason = reason;
   }
 };
+var OperationAbortedError = class extends DeepMemoryError {
+  operation;
+  constructor(operation) {
+    super(
+      "OPERATION_ABORTED",
+      `${operation} aborted by caller`,
+      `The caller's AbortSignal was triggered. Partial state from completed batches has been retained.`
+    );
+    this.name = "OperationAbortedError";
+    this.operation = operation;
+  }
+};
 var EmbeddingProviderRequiredError = class extends DeepMemoryError {
   constructor() {
     super(
@@ -821,10 +833,15 @@ var EntityManager = class {
         attempt++;
         if (attempt > maxRetries) {
           const errorMsg = err instanceof Error ? err.message : String(err);
+          const failureMessage = `embedBatch failed after ${maxRetries} retries: ${errorMsg}`;
+          const failed = entries.map(([id]) => ({ entityId: id, error: failureMessage }));
+          for (const { entityId, error } of failed) {
+            await options?.onItemFailed?.(entityId, error);
+          }
           return {
             processed: 0,
             failed: entries.length,
-            errors: entries.map(([id]) => ({ entityId: id, error: `embedBatch failed after ${maxRetries} retries: ${errorMsg}` })),
+            errors: failed,
             modelId: this.embedding.modelId(),
             dimensions: this.embedding.dimensions()
           };
@@ -847,7 +864,9 @@ var EntityManager = class {
         });
         processed++;
       } catch (err) {
-        errors.push({ entityId: id, error: err instanceof Error ? err.message : String(err) });
+        const errorMessage = err instanceof Error ? err.message : String(err);
+        errors.push({ entityId: id, error: errorMessage });
+        await options?.onItemFailed?.(id, errorMessage);
       }
     }
     return {
@@ -872,6 +891,7 @@ var EntityManager = class {
     }
     const batchSize = options?.batchSize ?? 50;
     const errorThreshold = options?.errorThresholdToAbort;
+    const signal = options?.signal;
     const firstPage = await this.storage.findEntities(this.repositoryId, { limit: 1, offset: 0 });
     const total = firstPage.total;
     let totalProcessed = 0;
@@ -879,6 +899,9 @@ var EntityManager = class {
     const allErrors = [];
     let offset = 0;
     while (offset < total) {
+      if (signal?.aborted) {
+        throw new OperationAbortedError("reembedAll");
+      }
       const page = await this.storage.findEntities(this.repositoryId, {
         limit: batchSize,
         offset
@@ -886,17 +909,21 @@ var EntityManager = class {
       if (page.items.length === 0) break;
       const ids = page.items.map((e) => e.id);
       const result = await this.reembedEntities(ids, {
-        maxRetries: options?.maxRetries
+        maxRetries: options?.maxRetries,
+        onItemFailed: options?.onItemFailed
       });
       totalProcessed += result.processed;
       totalFailed += result.failed;
       allErrors.push(...result.errors);
-      options?.onProgress?.(totalProcessed, total, totalFailed);
+      await options?.onProgress?.(totalProcessed, total, totalFailed);
       if (errorThreshold !== void 0 && totalFailed >= errorThreshold) {
         allErrors.push({ entityId: "", error: `Aborted: error threshold of ${errorThreshold} reached (${totalFailed} failures)` });
         break;
       }
       offset += page.items.length;
+      if (signal?.aborted) {
+        throw new OperationAbortedError("reembedAll");
+      }
       const delayMs = options?.delayBetweenBatchesMs ?? 0;
       if (delayMs > 0 && offset < total) {
         await new Promise((resolve) => {
@@ -2263,7 +2290,13 @@ var MemoryRepository = class {
    * the repository's embedding configuration is updated to the new values before
    * re-embedding, so all subsequent entity writes use the new model/dimensionality.
    *
-   * Processes in batches with optional rate limiting. Emits reembed:* events for progress tracking.
+   * Processes in batches with optional rate limiting.
+   *
+   * Progress is delivered via the optional `onProgress` callback. Most callers
+   * should use {@link DeepMemory.reembedAll} instead, which wires this callback
+   * to the global event bus so `reembed:*` events are reachable via
+   * `DeepMemory.on()`.
+   *
    * Updates repository metadata with the new model ID and dimensions on completion.
    */
   async reembedAll(options) {
@@ -2298,46 +2331,26 @@ var MemoryRepository = class {
         }
       });
     }
-    const stats = await this.getStats();
-    await this.eventBus.emit("reembed:started", {
-      repositoryId: this.repositoryId,
-      totalEntities: stats.entityCount
+    const result = await this.entityManager.reembedAll({
+      batchSize: options?.batchSize,
+      maxRetries: options?.maxRetries,
+      errorThresholdToAbort: options?.errorThresholdToAbort,
+      delayBetweenBatchesMs: options?.delayBetweenBatchesMs,
+      signal: options?.signal,
+      onProgress: async (processed, total, failed) => {
+        await options?.onProgress?.({ processed, totalEntities: total, failed });
+      },
+      onItemFailed: async (entityId, error) => {
+        await options?.onItemFailed?.({ entityId, error });
+      }
     });
-    try {
-      const result = await this.entityManager.reembedAll({
-        batchSize: options?.batchSize,
-        maxRetries: options?.maxRetries,
-        errorThresholdToAbort: options?.errorThresholdToAbort,
-        delayBetweenBatchesMs: options?.delayBetweenBatchesMs,
-        onProgress: async (processed, total, failed) => {
-          await this.eventBus.emit("reembed:progress", {
-            repositoryId: this.repositoryId,
-            processed,
-            totalEntities: total,
-            failed
-          });
-        }
-      });
-      await this.storage.updateRepository(this.repositoryId, {
-        metadata: {
-          embeddingModelId: result.modelId,
-          embeddingDimensions: result.dimensions
-        }
-      });
-      await this.eventBus.emit("reembed:completed", {
-        repositoryId: this.repositoryId,
-        processed: result.processed,
-        failed: result.failed,
-        modelId: result.modelId
-      });
-      return result;
-    } catch (err) {
-      await this.eventBus.emit("reembed:failed", {
-        repositoryId: this.repositoryId,
-        error: err instanceof Error ? err.message : String(err)
-      });
-      throw err;
-    }
+    await this.storage.updateRepository(this.repositoryId, {
+      metadata: {
+        embeddingModelId: result.modelId,
+        embeddingDimensions: result.dimensions
+      }
+    });
+    return result;
   }
   // ─── Relationships ─────────────────────────────────────────────────
   async createRelationships(inputs) {
@@ -3451,7 +3464,7 @@ var EventBus = class {
 };
 
 // src/portability/RepositoryExporter.ts
-var LIBRARY_VERSION = true ? "0.14.0" : "0.1.0";
+var LIBRARY_VERSION = true ? "0.16.0" : "0.1.0";
 var RepositoryExporter = class {
   storage;
   provenance;
@@ -3805,11 +3818,20 @@ var RepositoryImporter = class {
   storage;
   actorId;
   onProgress;
+  onItemFailed;
+  signal;
   migrationEngine = new MigrationEngine();
   constructor(config) {
     this.storage = config.storage;
     this.actorId = config.actorId;
     this.onProgress = config.onProgress;
+    this.onItemFailed = config.onItemFailed;
+    this.signal = config.signal;
+  }
+  throwIfAborted() {
+    if (this.signal?.aborted) {
+      throw new OperationAbortedError("import");
+    }
   }
   /**
    * Import an export archive according to the provided options.
@@ -3885,6 +3907,7 @@ var RepositoryImporter = class {
     let chunksCompleted = 0;
     const adaptiveConcurrencyHandle = options.bulk?.adaptiveConcurrencyHandle ?? {};
     for await (const chunk of chunks) {
+      this.throwIfAborted();
       const bulkResult = await this.storage.importBulk(target.repositoryId, [chunk], {
         ...options.bulk,
         skipExistenceCheck: true,
@@ -3899,6 +3922,8 @@ var RepositoryImporter = class {
           message: `Failed to import ${e.item}: ${e.error}`,
           id: e.item
         });
+        const itemType = chunk.entities?.some((x) => x.id === e.item) ? "entity" : chunk.relationships?.some((x) => x.id === e.item) ? "relationship" : "entity";
+        await this.onItemFailed?.({ itemId: e.item, itemType, error: e.error });
       }
       await this.onProgress?.({
         repositoryId: target.repositoryId,
@@ -3966,6 +3991,7 @@ var RepositoryImporter = class {
     let relationshipsSkipped = 0;
     let chunksCompleted = 0;
     for await (const chunk of chunks) {
+      this.throwIfAborted();
       if (chunk.entities) {
         for (const entity of chunk.entities) {
           let existing = await this.storage.getEntity(repositoryId, entity.id);
@@ -4034,11 +4060,13 @@ var RepositoryImporter = class {
             const targetExists = await this.storage.getEntity(repositoryId, rel.targetEntityId);
             if (!sourceExists || !targetExists) {
               relationshipsSkipped++;
+              const errorMsg = `source or target entity missing`;
               warnings.push({
                 code: "relationship_orphaned",
-                message: `Relationship "${rel.id}" skipped \u2014 source or target entity missing`,
+                message: `Relationship "${rel.id}" skipped \u2014 ${errorMsg}`,
                 relationshipId: rel.id
               });
+              await this.onItemFailed?.({ itemId: rel.id, itemType: "relationship", error: errorMsg });
             } else {
               try {
                 await this.storage.createRelationship(repositoryId, rel);
@@ -4384,22 +4412,37 @@ var DeepMemory = class {
     const importer = new RepositoryImporter({
       storage: this.storage,
       actorId: this.provenance.getContext().actorId,
-      onProgress: (progress) => this.globalEventBus.emit("import:progress", progress)
+      onProgress: (progress) => this.globalEventBus.emit("import:progress", progress),
+      onItemFailed: (failure) => this.globalEventBus.emit("import:item-failed", {
+        repositoryId: options.target.repositoryId,
+        itemId: failure.itemId,
+        itemType: failure.itemType,
+        error: failure.error
+      }),
+      signal: options.signal
     });
-    const result = await importer.import(archive, options);
-    if (result.success) {
-      await this.globalEventBus.emit("import:completed", {
-        repositoryId: result.repositoryId,
-        entitiesImported: result.statistics.entitiesImported,
-        relationshipsImported: result.statistics.relationshipsImported
-      });
-    } else {
+    try {
+      const result = await importer.import(archive, options);
+      if (result.success) {
+        await this.globalEventBus.emit("import:completed", {
+          repositoryId: result.repositoryId,
+          entitiesImported: result.statistics.entitiesImported,
+          relationshipsImported: result.statistics.relationshipsImported
+        });
+      } else {
+        await this.globalEventBus.emit("import:failed", {
+          repositoryId: result.repositoryId,
+          error: result.warnings.map((w) => w.message).join("; ")
+        });
+      }
+      return result;
+    } catch (err) {
       await this.globalEventBus.emit("import:failed", {
-        repositoryId: result.repositoryId,
-        error: result.warnings.map((w) => w.message).join("; ")
+        repositoryId: options.target.repositoryId,
+        error: err instanceof Error ? err.message : String(err)
       });
+      throw err;
     }
-    return result;
   }
   /**
    * Stream a repository export as an async generator.
@@ -4468,23 +4511,88 @@ var DeepMemory = class {
     const importer = new RepositoryImporter({
       storage: this.storage,
       actorId: this.provenance.getContext().actorId,
-      onProgress: (progress) => this.globalEventBus.emit("import:progress", progress)
+      onProgress: (progress) => this.globalEventBus.emit("import:progress", progress),
+      onItemFailed: (failure) => this.globalEventBus.emit("import:item-failed", {
+        repositoryId: options.target.repositoryId,
+        itemId: failure.itemId,
+        itemType: failure.itemType,
+        error: failure.error
+      }),
+      signal: options.signal
     });
     await this.globalEventBus.emit("import:started", { repositoryId: options.target.repositoryId });
-    const result = await importer.importStream(header, chunks, options);
-    if (result.success) {
-      await this.globalEventBus.emit("import:completed", {
-        repositoryId: result.repositoryId,
-        entitiesImported: result.statistics.entitiesImported,
-        relationshipsImported: result.statistics.relationshipsImported
-      });
-    } else {
+    try {
+      const result = await importer.importStream(header, chunks, options);
+      if (result.success) {
+        await this.globalEventBus.emit("import:completed", {
+          repositoryId: result.repositoryId,
+          entitiesImported: result.statistics.entitiesImported,
+          relationshipsImported: result.statistics.relationshipsImported
+        });
+      } else {
+        await this.globalEventBus.emit("import:failed", {
+          repositoryId: result.repositoryId,
+          error: result.warnings.map((w) => w.message).join("; ")
+        });
+      }
+      return result;
+    } catch (err) {
       await this.globalEventBus.emit("import:failed", {
-        repositoryId: result.repositoryId,
-        error: result.warnings.map((w) => w.message).join("; ")
+        repositoryId: options.target.repositoryId,
+        error: err instanceof Error ? err.message : String(err)
       });
+      throw err;
+    }
+  }
+  /**
+   * Re-embed all entities in a repository, emitting `reembed:*` events on the
+   * global event bus (reachable via {@link DeepMemory.on}).
+   *
+   * Mirrors the lifecycle of {@link importRepositoryStream}: emits
+   * `reembed:started` up front, `reembed:progress` after every batch,
+   * and either `reembed:completed` or `reembed:failed` at the end.
+   *
+   * `options.signal` is honoured at batch boundaries — any vectors already
+   * written by completed batches are retained, and `reembed:failed` is emitted
+   * with the abort error message.
+   */
+  async reembedAll(repositoryId, options) {
+    await this.ensureInitialized();
+    const repo = await this.openRepository(repositoryId);
+    const stats = await repo.getStats();
+    await this.globalEventBus.emit("reembed:started", {
+      repositoryId,
+      totalEntities: stats.entityCount
+    });
+    try {
+      const result = await repo.reembedAll({
+        ...options,
+        onProgress: (progress) => this.globalEventBus.emit("reembed:progress", {
+          repositoryId,
+          processed: progress.processed,
+          totalEntities: progress.totalEntities,
+          failed: progress.failed
+        }),
+        onItemFailed: (failure) => this.globalEventBus.emit("reembed:item-failed", {
+          repositoryId,
+          entityId: failure.entityId,
+          error: failure.error
+        })
+      });
+      await this.globalEventBus.emit("reembed:completed", {
+        repositoryId,
+        processed: result.processed,
+        failed: result.failed,
+        modelId: result.modelId
+      });
+      return result;
+    } catch (err) {
+      await this.globalEventBus.emit("reembed:failed", {
+        repositoryId,
+        error: err instanceof Error ? err.message : String(err)
+      });
+      throw err;
     }
-    return result;
   }
   /** Subscribe to global events */
   on(event, handler) {
@@ -5585,6 +5693,7 @@ export {
   InvalidInputError,
   MemoryRepository,
   NoOpEmbeddingProvider,
+  OperationAbortedError,
   OperationCancelledError,
   ProviderError,
   RelationshipConstraintError,