@twin.org/auditable-item-stream-service 0.0.3-next.9 → 0.9.0

package/dist/es/auditableItemStreamService.js

@@ -1,13 +1,15 @@
 // Copyright 2024 IOTA Stiftung.
 // SPDX-License-Identifier: Apache-2.0.
-import { AuditableItemStreamContexts, AuditableItemStreamTopics, AuditableItemStreamTypes } from "@twin.org/auditable-item-stream-models";
-import { ContextIdKeys, ContextIdStore } from "@twin.org/context";
-import { Coerce, ComponentFactory, GeneralError, Guards, Is, NotFoundError, ObjectHelper, RandomHelper, Urn, Validation } from "@twin.org/core";
-import { JsonLdHelper, JsonLdProcessor } from "@twin.org/data-json-ld";
+import { AuditableItemStreamContexts, AuditableItemStreamDataTypes, AuditableItemStreamMetricIds, AuditableItemStreamMetrics, AuditableItemStreamModes, AuditableItemStreamTopics, AuditableItemStreamTypes } from "@twin.org/auditable-item-stream-models";
+import { ContextIdHelper, ContextIdKeys, ContextIdStore } from "@twin.org/context";
+import { Coerce, ComponentFactory, GeneralError, Guards, Is, Mutex, NotFoundError, ObjectHelper, RandomHelper, Urn, Validation } from "@twin.org/core";
+import { DataTypeHelper } from "@twin.org/data-core";
+import { JsonLdDataTypes, JsonLdHelper, JsonLdProcessor } from "@twin.org/data-json-ld";
 import { ComparisonOperator, LogicalOperator, SortDirection } from "@twin.org/entity";
 import { EntityStorageConnectorFactory } from "@twin.org/entity-storage-models";
-import { ImmutableProofContexts } from "@twin.org/immutable-proof-models";
+import { ImmutableProofContexts, ImmutableProofDataTypes } from "@twin.org/immutable-proof-models";
 import { SchemaOrgContexts, SchemaOrgDataTypes, SchemaOrgTypes } from "@twin.org/standards-schema-org";
+import { MetricHelper } from "@twin.org/telemetry-models";
 /**
  * Class for performing auditable item stream operations.
  */
@@ -68,11 +70,21 @@ export class AuditableItemStreamService {
      * @internal
      */
     _eventBusComponent;
+    /**
+     * The telemetry component.
+     * @internal
+     */
+    _telemetryComponent;
     /**
      * The default interval for the integrity checks.
      * @internal
      */
     _defaultImmutableInterval;
+    /**
+     * The timeout in milliseconds when acquiring a mutex lock.
+     * @internal
+     */
+    _mutexTimeoutMs;
     /**
      * Create a new instance of AuditableItemStreamService.
      * @param options The dependencies for the auditable item stream connector.
@@ -84,9 +96,14 @@ export class AuditableItemStreamService {
         if (Is.stringValue(options?.eventBusComponentType)) {
             this._eventBusComponent = ComponentFactory.get(options.eventBusComponentType);
         }
+        this._telemetryComponent = ComponentFactory.getIfExists(options?.telemetryComponentType);
         this._config = options?.config ?? {};
         this._defaultImmutableInterval = this._config.defaultImmutableInterval ?? 10;
+        this._mutexTimeoutMs = Coerce.integer(options?.config?.mutexTimeoutMs);
         SchemaOrgDataTypes.registerRedirects();
+        AuditableItemStreamDataTypes.registerTypes();
+        JsonLdDataTypes.registerTypes();
+        ImmutableProofDataTypes.registerTypes();
     }
     /**
      * Returns the class name of the component.
@@ -95,49 +112,59 @@ export class AuditableItemStreamService {
     className() {
         return AuditableItemStreamService.CLASS_NAME;
     }
+    /**
+     * Register all AIS metrics with the telemetry component.
+     * @returns A promise that resolves when the metrics have been registered.
+     */
+    async start() {
+        if (Is.undefined(this._telemetryComponent)) {
+            return;
+        }
+        await MetricHelper.createMetrics(this._telemetryComponent, AuditableItemStreamMetrics);
+    }
     /**
      * Create a new stream.
      * @param stream The stream to create.
-     * @param stream.annotationObject The object for the stream as JSON-LD.
-     * @param stream.entries Entries to store in the stream.
-     * @param options Options for creating the stream.
-     * @param options.immutableInterval After how many entries do we add immutable checks, defaults to service configured value.
-     * A value of 0 will disable integrity checks, 1 will be every item, or any other integer for an interval.
      * @returns The id of the new stream item.
      */
-    async create(stream, options) {
+    async create(stream) {
         Guards.object(AuditableItemStreamService.CLASS_NAME, "stream", stream);
         const contextIds = await ContextIdStore.getContextIds();
+        ContextIdHelper.guard(contextIds, ContextIdKeys.Organization);
         try {
+            const ownerOrganizationId = contextIds[ContextIdKeys.UserOrganization] ?? contextIds[ContextIdKeys.Organization];
+            const id = RandomHelper.generateUuidV7("compact");
+            const schemaValidationFailures = [];
+            await DataTypeHelper.validate("stream", `${AuditableItemStreamContexts.Namespace}${AuditableItemStreamTypes.Stream}Base`, stream, schemaValidationFailures);
+            Validation.asValidationError(AuditableItemStreamService.CLASS_NAME, "stream", schemaValidationFailures);
+            if (stream.closed && !Is.arrayValue(stream.entries?.[SchemaOrgTypes.ItemListElement])) {
+                throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "closedRequiresEntries");
+            }
             if (Is.object(stream.annotationObject)) {
                 const validationFailures = [];
                 await JsonLdHelper.validate(stream.annotationObject, validationFailures);
                 Validation.asValidationError(AuditableItemStreamService.CLASS_NAME, "stream.annotationObject", validationFailures);
             }
-            const id = RandomHelper.generateUuidV7("compact");
             const context = {
                 now: new Date(Date.now()).toISOString(),
                 contextIds,
                 indexCounter: 0,
-                immutableInterval: options?.immutableInterval ?? this._defaultImmutableInterval
+                immutableInterval: stream?.immutableInterval ?? this._defaultImmutableInterval,
+                organizationIdentity: ownerOrganizationId
             };
             const streamEntity = {
                 id,
-                organizationIdentity: contextIds?.[ContextIdKeys.Organization],
+                organizationIdentity: ownerOrganizationId,
                 userIdentity: contextIds?.[ContextIdKeys.User],
                 dateCreated: context.now,
                 immutableInterval: context.immutableInterval,
+                closed: stream.closed,
+                mode: stream.mode,
                 numberOfItems: 0
             };
-            // Create the JSON-LD object we want to use for the proof
-            // this is a subset of fixed properties from the stream object.
-            const streamModel = this.streamEntityToJsonLd(ObjectHelper.pick(streamEntity, AuditableItemStreamService._PROOF_KEYS_STREAM));
-            // Create the proof for the stream object
-            if (context.immutableInterval > 0) {
-                streamEntity.proofId = await this._immutableProofComponent.create(streamModel);
-            }
-            if (Is.arrayValue(stream.entries)) {
-                for (const entry of stream.entries) {
+            const streamUrn = await this.createStreamProof(streamEntity, context.immutableInterval);
+            if (Is.arrayValue(stream.entries?.[SchemaOrgTypes.ItemListElement])) {
+                for (const entry of stream.entries[SchemaOrgTypes.ItemListElement]) {
                     await this.setEntry(context, id, entry);
                 }
             }
@@ -146,25 +173,23 @@ export class AuditableItemStreamService {
             streamEntity.annotationObject = stream.annotationObject;
             streamEntity.numberOfItems = context.indexCounter;
             await this._streamStorage.set(streamEntity);
-            await this._eventBusComponent?.publish(AuditableItemStreamTopics.StreamCreated, { id: streamModel.id });
-            return streamModel.id;
+            await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.StreamsCreated, {
+                mode: streamEntity.mode ?? AuditableItemStreamModes.Default,
+                immutableInterval: context.immutableInterval
+            });
+            await this._eventBusComponent?.publish(AuditableItemStreamTopics.StreamCreated, { id: streamUrn });
+            return streamUrn;
         }
         catch (error) {
             throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "createFailed", undefined, error);
         }
     }
     /**
-     * Get a stream header without the entries.
-     * @param id The id of the stream to get.
-     * @param options Additional options for the get operation.
-     * @param options.includeEntries Whether to include the entries, defaults to false.
-     * @param options.includeDeleted Whether to include deleted entries, defaults to false.
-     * @param options.verifyStream Should the stream be verified, defaults to false.
-     * @param options.verifyEntries Should the entries be verified, defaults to false.
-     * @returns The stream and entries if found.
-     * @throws NotFoundError if the stream is not found
+     * Close a stream.
+     * @param id The id of the stream to close.
+     * @returns A promise that resolves when the stream has been closed.
      */
-    async get(id, options) {
+    async close(id) {
         Guards.stringValue(AuditableItemStreamService.CLASS_NAME, "id", id);
         const urnParsed = Urn.fromValidString(id);
         if (urnParsed.namespaceIdentifier() !== AuditableItemStreamService._NAMESPACE) {
@@ -173,39 +198,32 @@ export class AuditableItemStreamService {
                 id
             });
         }
+        const streamId = urnParsed.namespaceSpecific(0);
+        await Mutex.lock(streamId, { throwOnTimeout: true, timeoutMs: this._mutexTimeoutMs });
         try {
-            const streamId = urnParsed.namespaceSpecific(0);
             const streamEntity = await this._streamStorage.get(streamId);
             if (Is.empty(streamEntity)) {
                 throw new NotFoundError(AuditableItemStreamService.CLASS_NAME, "streamNotFound", id);
             }
-            const verifyStream = options?.verifyStream ?? false;
-            const verifyEntries = options?.verifyEntries ?? false;
-            const streamModel = this.streamEntityToJsonLd(streamEntity);
-            if (options?.includeEntries) {
-                const result = await this.findEntries(streamId, options?.includeDeleted, verifyEntries);
-                streamModel.entries = result.entries;
-                streamModel.cursor = result.cursor;
-            }
-            if (verifyStream && Is.stringValue(streamEntity.proofId)) {
-                streamModel.verification = await this._immutableProofComponent.verify(streamEntity.proofId);
-            }
-            if (verifyStream || verifyEntries) {
-                streamModel["@context"].push(ImmutableProofContexts.Context);
+            if (!streamEntity.closed) {
+                streamEntity.closed = true;
+                streamEntity.dateModified = new Date(Date.now()).toISOString();
+                await this._streamStorage.set(streamEntity);
+                await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.StreamsClosed);
+                await this._eventBusComponent?.publish(AuditableItemStreamTopics.StreamUpdated, { id });
             }
-            const result = await JsonLdProcessor.compact(streamModel, streamModel["@context"]);
-            return result;
         }
         catch (error) {
-            throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "getFailed", undefined, error);
+            throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "closeFailed", undefined, error);
+        }
+        finally {
+            Mutex.unlock(streamId);
         }
     }
     /**
      * Update a stream.
-     * @param stream The stream to update.
-     * @param stream.id The id of the stream to update.
-     * @param stream.annotationObject The object for the stream as JSON-LD.
-     * @returns Nothing.
+     * @param stream The stream to update, does not update entries.
+     * @returns A promise that resolves when the stream has been updated.
      */
     async update(stream) {
         Guards.object(AuditableItemStreamService.CLASS_NAME, "stream", stream);
@@ -217,8 +235,12 @@ export class AuditableItemStreamService {
                 id: stream.id
             });
         }
+        const streamId = urnParsed.namespaceSpecific(0);
+        await Mutex.lock(streamId, { throwOnTimeout: true, timeoutMs: this._mutexTimeoutMs });
         try {
-            const streamId = urnParsed.namespaceSpecific(0);
+            const schemaValidationFailures = [];
+            await DataTypeHelper.validate("stream", `${AuditableItemStreamContexts.Namespace}${AuditableItemStreamTypes.Stream}`, stream, schemaValidationFailures);
+            Validation.asValidationError(AuditableItemStreamService.CLASS_NAME, "stream", schemaValidationFailures);
             const streamEntity = await this._streamStorage.get(streamId);
             if (Is.empty(streamEntity)) {
                 throw new NotFoundError(AuditableItemStreamService.CLASS_NAME, "streamNotFound", stream.id);
@@ -228,21 +250,99 @@ export class AuditableItemStreamService {
                 await JsonLdHelper.validate(stream.annotationObject, validationFailures);
                 Validation.asValidationError(AuditableItemStreamService.CLASS_NAME, "stream.annotationObject", validationFailures);
             }
+            let changed = false;
             if (!ObjectHelper.equal(streamEntity.annotationObject, stream.annotationObject, false)) {
                 streamEntity.annotationObject = stream.annotationObject;
+                changed = true;
+            }
+            const contextIds = await ContextIdStore.getContextIds();
+            const ownerOrganizationId = contextIds?.[ContextIdKeys.UserOrganization] ?? contextIds?.[ContextIdKeys.Organization];
+            if (!Is.stringValue(streamEntity.organizationIdentity) &&
+                Is.stringValue(ownerOrganizationId)) {
+                streamEntity.organizationIdentity = ownerOrganizationId;
+                changed = true;
+            }
+            if (!Is.stringValue(streamEntity.proofId) &&
+                Is.stringValue(streamEntity.organizationIdentity)) {
+                await this.createStreamProof(streamEntity, streamEntity.immutableInterval ?? this._defaultImmutableInterval);
+                if (Is.stringValue(streamEntity.proofId)) {
+                    changed = true;
+                }
+            }
+            if (changed) {
                 streamEntity.dateModified = new Date(Date.now()).toISOString();
                 await this._streamStorage.set(streamEntity);
+                await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.StreamsUpdated);
                 await this._eventBusComponent?.publish(AuditableItemStreamTopics.StreamUpdated, { id: stream.id });
             }
         }
         catch (error) {
             throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "updatingFailed", undefined, error);
         }
+        finally {
+            Mutex.unlock(streamId);
+        }
+    }
+    /**
+     * Get a stream header without the entries.
+     * @param id The id of the stream to get.
+     * @param cursor Cursor to use for next chunk of entries.
+     * @param limit Limit the number of entries to return, only applicable if includeEntries is true.
+     * @param options Additional options for the get operation.
+     * @param options.includeEntries Whether to include the entries, defaults to false.
+     * @param options.includeDeleted Whether to include deleted entries, defaults to false.
+     * @param options.verifyStream Should the stream be verified, defaults to false.
+     * @param options.verifyEntries Should the entries be verified, defaults to false.
+     * @returns The stream and entries if found.
+     * @throws NotFoundError if the stream is not found
+     */
+    async get(id, cursor, limit, options) {
+        Guards.stringValue(AuditableItemStreamService.CLASS_NAME, "id", id);
+        const urnParsed = Urn.fromValidString(id);
+        if (urnParsed.namespaceIdentifier() !== AuditableItemStreamService._NAMESPACE) {
+            throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "namespaceMismatch", {
+                namespace: AuditableItemStreamService._NAMESPACE,
+                id
+            });
+        }
+        try {
+            const streamId = urnParsed.namespaceSpecific(0);
+            const streamEntity = await this._streamStorage.get(streamId);
+            if (Is.empty(streamEntity)) {
+                throw new NotFoundError(AuditableItemStreamService.CLASS_NAME, "streamNotFound", id);
+            }
+            const verifyStream = options?.verifyStream ?? false;
+            const verifyEntries = options?.verifyEntries ?? false;
+            const streamModel = this.streamEntityToJsonLd(streamEntity);
+            let returnCursor;
+            if (options?.includeEntries) {
+                const result = await this.findEntries(streamId, options?.includeDeleted, verifyEntries, undefined, undefined, undefined, limit, cursor);
+                streamModel.entries = {
+                    type: SchemaOrgTypes.ItemList,
+                    [SchemaOrgTypes.ItemListElement]: result.entries
+                };
+                returnCursor = result.cursor;
+            }
+            if (verifyStream && Is.stringValue(streamEntity.proofId)) {
+                streamModel.verification = await this._immutableProofComponent.verify(streamEntity.proofId);
+            }
+            if (verifyStream || verifyEntries) {
+                streamModel["@context"].push(ImmutableProofContexts.Context);
+            }
+            const result = await JsonLdProcessor.compact(streamModel, streamModel["@context"]);
+            return {
+                stream: result,
+                cursor: returnCursor
+            };
+        }
+        catch (error) {
+            throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "getFailed", undefined, error);
+        }
     }
     /**
      * Delete the stream.
      * @param id The id of the stream to remove.
-     * @returns Nothing.
+     * @returns A promise that resolves when the stream has been removed.
      */
     async remove(id) {
         Guards.stringValue(AuditableItemStreamService.CLASS_NAME, "id", id);
@@ -253,19 +353,24 @@ export class AuditableItemStreamService {
                 id
             });
         }
+        const streamId = urnParsed.namespaceSpecific(0);
+        await Mutex.lock(streamId, { throwOnTimeout: true, timeoutMs: this._mutexTimeoutMs });
         try {
-            const streamId = urnParsed.namespaceSpecific(0);
             const streamEntity = await this._streamStorage.get(streamId);
             if (Is.empty(streamEntity)) {
                 throw new NotFoundError(AuditableItemStreamService.CLASS_NAME, "streamNotFound", id);
             }
             await this.internalRemoveEntries(streamEntity, false);
             await this._streamStorage.remove(streamEntity.id);
+            await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.StreamsDeleted);
             await this._eventBusComponent?.publish(AuditableItemStreamTopics.StreamDeleted, { id });
         }
         catch (error) {
             throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "removingFailed", undefined, error);
         }
+        finally {
+            Mutex.unlock(streamId);
+        }
     }
     /**
      * Query all the streams, will not return entries.
@@ -339,17 +444,25 @@ export class AuditableItemStreamService {
                 id: streamId
             });
         }
+        const streamIdParts = urnParsed.namespaceSpecific(0);
+        await Mutex.lock(streamIdParts, { throwOnTimeout: true, timeoutMs: this._mutexTimeoutMs });
         try {
-            const streamIdParts = urnParsed.namespaceSpecific(0);
             const streamEntity = await this._streamStorage.get(streamIdParts);
             if (Is.empty(streamEntity)) {
                 throw new NotFoundError(AuditableItemStreamService.CLASS_NAME, "streamNotFound", streamIdParts);
             }
+            if (streamEntity.closed) {
+                await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.ClosedStreamRejections, { operation: "createEntry" });
+                throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "streamClosed", {
+                    id: streamId
+                });
+            }
             const context = {
                 now: new Date(Date.now()).toISOString(),
                 contextIds,
                 indexCounter: streamEntity.numberOfItems,
-                immutableInterval: streamEntity.immutableInterval
+                immutableInterval: streamEntity.immutableInterval,
+                organizationIdentity: streamEntity.organizationIdentity ?? contextIds?.[ContextIdKeys.Organization]
             };
             const createdId = await this.setEntry(context, streamEntity.id, {
                 entryObject
@@ -357,6 +470,10 @@ export class AuditableItemStreamService {
             streamEntity.dateModified = context.now;
             streamEntity.numberOfItems = context.indexCounter;
             await this._streamStorage.set(streamEntity);
+            await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.EntriesCreated, {
+                hasProof: context.immutableInterval > 0 &&
+                    (context.indexCounter - 1) % context.immutableInterval === 0
+            });
             const fullId = new Urn(AuditableItemStreamService._NAMESPACE, [
                 streamEntity.id,
                 createdId
@@ -367,6 +484,9 @@ export class AuditableItemStreamService {
         catch (error) {
             throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "creatingEntryFailed", undefined, error);
         }
+        finally {
+            Mutex.unlock(streamIdParts);
+        }
     }
     /**
      * Get the entry from the stream.
@@ -408,7 +528,7 @@ export class AuditableItemStreamService {
             }
             const entry = this.streamEntryEntityToJsonLd(result.entity);
             if (verifyEntry) {
-                entry["@context"].push(ImmutableProofContexts.Context);
+                entry["@context"] = JsonLdProcessor.combineContexts(entry["@context"], ImmutableProofContexts.Context);
                 entry.verification = result.verification;
             }
             const result2 = await JsonLdProcessor.compact(entry, entry["@context"]);
@@ -464,12 +584,11 @@ export class AuditableItemStreamService {
      * @param streamId The id of the stream to update.
      * @param entryId The id of the entry to update.
      * @param entryObject The object for the entry as JSON-LD.
-     * @returns Nothing.
+     * @returns A promise that resolves when the entry has been updated.
      */
     async updateEntry(streamId, entryId, entryObject) {
         Guards.stringValue(AuditableItemStreamService.CLASS_NAME, "streamId", streamId);
         Guards.stringValue(AuditableItemStreamService.CLASS_NAME, "entryId", entryId);
-        const contextIds = await ContextIdStore.getContextIds();
         const urnParsed = Urn.fromValidString(streamId);
         if (urnParsed.namespaceIdentifier() !== AuditableItemStreamService._NAMESPACE) {
             throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "namespaceMismatch", {
@@ -484,8 +603,9 @@ export class AuditableItemStreamService {
                 id: entryId
             });
         }
+        const streamNamespaceId = urnParsed.namespaceSpecific(0);
+        await Mutex.lock(streamNamespaceId, { throwOnTimeout: true, timeoutMs: this._mutexTimeoutMs });
         try {
-            const streamNamespaceId = urnParsed.namespaceMethod();
             const streamEntryNamespaceId = urnParsedEntry.namespaceMethod();
             if (streamNamespaceId !== streamEntryNamespaceId) {
                 throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "namespaceMismatch", {
@@ -497,16 +617,31 @@ export class AuditableItemStreamService {
             if (Is.empty(streamEntity)) {
                 throw new NotFoundError(AuditableItemStreamService.CLASS_NAME, "streamNotFound", streamId);
             }
+            if (streamEntity.closed) {
+                await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.ClosedStreamRejections, { operation: "updateEntry" });
+                throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "streamClosed", {
+                    id: streamId
+                });
+            }
+            if (streamEntity.mode === AuditableItemStreamModes.AppendOnly) {
+                await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.AppendOnlyRejections, { operation: "updateEntry" });
+                throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "appendOnlyNoEntryUpdates", {
+                    id: streamId
+                });
+            }
             const entryNamespaceId = urnParsedEntry.namespaceSpecific(1);
             const existing = await this.findEntry(streamEntity.id, entryNamespaceId);
             if (Is.empty(existing)) {
                 throw new NotFoundError(AuditableItemStreamService.CLASS_NAME, "streamEntryNotFound", entryId);
             }
+            const contextIds = await ContextIdStore.getContextIds();
+            const ownerOrganizationId = contextIds?.[ContextIdKeys.UserOrganization] ?? contextIds?.[ContextIdKeys.Organization];
             const context = {
                 now: new Date(Date.now()).toISOString(),
                 contextIds,
                 indexCounter: streamEntity.numberOfItems,
-                immutableInterval: streamEntity.immutableInterval
+                immutableInterval: streamEntity.immutableInterval,
+                organizationIdentity: streamEntity.organizationIdentity ?? ownerOrganizationId
             };
             await this.setEntry(context, streamEntity.id, {
                 ...existing.entity,
@@ -515,22 +650,25 @@ export class AuditableItemStreamService {
             streamEntity.dateModified = context.now;
             streamEntity.numberOfItems = context.indexCounter;
             await this._streamStorage.set(streamEntity);
+            await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.EntriesUpdated);
             await this._eventBusComponent?.publish(AuditableItemStreamTopics.StreamEntryUpdated, { id: streamId, entryId });
         }
         catch (error) {
             throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "updatingEntryFailed", undefined, error);
         }
+        finally {
+            Mutex.unlock(streamNamespaceId);
+        }
     }
     /**
      * Delete from the stream.
      * @param streamId The id of the stream to remove from.
      * @param entryId The id of the entry to remove.
-     * @returns Nothing.
+     * @returns A promise that resolves when the entry has been removed.
      */
     async removeEntry(streamId, entryId) {
         Guards.stringValue(AuditableItemStreamService.CLASS_NAME, "streamId", streamId);
         Guards.stringValue(AuditableItemStreamService.CLASS_NAME, "entryId", entryId);
-        const contextIds = await ContextIdStore.getContextIds();
         const urnParsed = Urn.fromValidString(streamId);
         if (urnParsed.namespaceIdentifier() !== AuditableItemStreamService._NAMESPACE) {
             throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "namespaceMismatch", {
@@ -545,8 +683,9 @@ export class AuditableItemStreamService {
                 id: entryId
             });
         }
+        const streamNamespaceId = urnParsed.namespaceSpecific(0);
+        await Mutex.lock(streamNamespaceId, { throwOnTimeout: true, timeoutMs: this._mutexTimeoutMs });
         try {
-            const streamNamespaceId = urnParsed.namespaceMethod();
             const streamEntryNamespaceId = urnParsedEntry.namespaceMethod();
             if (streamNamespaceId !== streamEntryNamespaceId) {
                 throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "namespaceMismatch", {
@@ -558,17 +697,26 @@ export class AuditableItemStreamService {
             if (Is.empty(streamEntity)) {
                 throw new NotFoundError(AuditableItemStreamService.CLASS_NAME, "streamNotFound", streamId);
             }
+            if (streamEntity.mode === AuditableItemStreamModes.AppendOnly) {
+                await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.AppendOnlyRejections, { operation: "removeEntry" });
+                throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "appendOnlyNoEntryRemovals", {
+                    id: streamId
+                });
+            }
             const entryNamespaceId = urnParsedEntry.namespaceSpecific(1);
             const result = await this.findEntry(streamNamespaceId, entryNamespaceId);
             if (Is.empty(result)) {
                 throw new NotFoundError(AuditableItemStreamService.CLASS_NAME, "streamEntryNotFound", entryId);
             }
             if (Is.empty(result.entity.dateDeleted)) {
+                const contextIds = await ContextIdStore.getContextIds();
+                const ownerOrganizationId = contextIds?.[ContextIdKeys.UserOrganization] ?? contextIds?.[ContextIdKeys.Organization];
                 const context = {
                     now: new Date(Date.now()).toISOString(),
                     contextIds,
                     indexCounter: streamEntity.numberOfItems,
-                    immutableInterval: streamEntity.immutableInterval
+                    immutableInterval: streamEntity.immutableInterval,
+                    organizationIdentity: streamEntity.organizationIdentity ?? ownerOrganizationId
                 };
                 await this.setEntry(context, streamEntity.id, {
                     ...result.entity,
@@ -577,16 +725,20 @@ export class AuditableItemStreamService {
                 streamEntity.dateModified = context.now;
                 streamEntity.numberOfItems = context.indexCounter;
                 await this._streamStorage.set(streamEntity);
+                await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.EntriesDeleted);
                 await this._eventBusComponent?.publish(AuditableItemStreamTopics.StreamEntryDeleted, { id: streamId, entryId });
             }
         }
         catch (error) {
             throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "removingEntryFailed", undefined, error);
         }
+        finally {
+            Mutex.unlock(streamNamespaceId);
+        }
     }
     /**
      * Get the entries for the stream.
-     * @param streamId The id of the stream to get.
+     * @param streamId The id of the stream to get, if undefined returns all matching entries.
      * @param options Additional options for the get operation.
      * @param options.conditions The conditions to filter the stream.
      * @param options.includeDeleted Whether to include deleted entries, defaults to false.
@@ -598,19 +750,24 @@ export class AuditableItemStreamService {
      * @throws NotFoundError if the stream is not found.
      */
     async getEntries(streamId, options) {
-        Guards.stringValue(AuditableItemStreamService.CLASS_NAME, "streamId", streamId);
-        const urnParsed = Urn.fromValidString(streamId);
-        if (urnParsed.namespaceIdentifier() !== AuditableItemStreamService._NAMESPACE) {
-            throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "namespaceMismatch", {
-                namespace: AuditableItemStreamService._NAMESPACE,
-                id: streamId
-            });
+        let streamNamespaceId;
+        if (!Is.empty(streamId)) {
+            Guards.stringValue(AuditableItemStreamService.CLASS_NAME, "streamId", streamId);
+            const urnParsed = Urn.fromValidString(streamId);
+            if (urnParsed.namespaceIdentifier() !== AuditableItemStreamService._NAMESPACE) {
+                throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "namespaceMismatch", {
+                    namespace: AuditableItemStreamService._NAMESPACE,
+                    id: streamId
+                });
+            }
+            streamNamespaceId = urnParsed.namespaceSpecific(0);
         }
         try {
-            const streamNamespaceId = urnParsed.namespaceSpecific(0);
-            const streamEntity = await this._streamStorage.get(streamNamespaceId);
-            if (Is.empty(streamEntity)) {
-                throw new NotFoundError(AuditableItemStreamService.CLASS_NAME, "streamNotFound", streamId);
+            if (Is.stringValue(streamNamespaceId)) {
+                const streamEntity = await this._streamStorage.get(streamNamespaceId);
+                if (Is.empty(streamEntity)) {
+                    throw new NotFoundError(AuditableItemStreamService.CLASS_NAME, "streamNotFound", streamId);
+                }
             }
             const verifyEntries = options?.verifyEntries ?? false;
             const result = await this.findEntries(streamNamespaceId, options?.includeDeleted, verifyEntries, options?.conditions, options?.order, undefined, options?.limit, options?.cursor);
@@ -638,7 +795,7 @@ export class AuditableItemStreamService {
     }
     /**
      * Get the entry objects for the stream.
-     * @param streamId The id of the stream to get.
+     * @param streamId The id of the stream to get, if undefined returns all matching entries.
      * @param options Additional options for the get operation.
      * @param options.conditions The conditions to filter the stream.
      * @param options.includeDeleted Whether to include deleted entries, defaults to false.
@@ -649,19 +806,24 @@ export class AuditableItemStreamService {
      * @throws NotFoundError if the stream is not found.
      */
     async getEntryObjects(streamId, options) {
-        Guards.stringValue(AuditableItemStreamService.CLASS_NAME, "streamId", streamId);
-        const urnParsed = Urn.fromValidString(streamId);
-        if (urnParsed.namespaceIdentifier() !== AuditableItemStreamService._NAMESPACE) {
-            throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "namespaceMismatch", {
-                namespace: AuditableItemStreamService._NAMESPACE,
-                id: streamId
-            });
+        let streamNamespaceId;
+        if (!Is.empty(streamId)) {
+            Guards.stringValue(AuditableItemStreamService.CLASS_NAME, "streamId", streamId);
+            const urnParsed = Urn.fromValidString(streamId);
+            if (urnParsed.namespaceIdentifier() !== AuditableItemStreamService._NAMESPACE) {
+                throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "namespaceMismatch", {
+                    namespace: AuditableItemStreamService._NAMESPACE,
+                    id: streamId
+                });
+            }
+            streamNamespaceId = urnParsed.namespaceSpecific(0);
         }
         try {
-            const streamNamespaceId = urnParsed.namespaceSpecific(0);
-            const streamEntity = await this._streamStorage.get(streamNamespaceId);
-            if (Is.empty(streamEntity)) {
-                throw new NotFoundError(AuditableItemStreamService.CLASS_NAME, "streamNotFound", streamId);
+            if (Is.stringValue(streamNamespaceId)) {
+                const streamEntity = await this._streamStorage.get(streamNamespaceId);
+                if (Is.empty(streamEntity)) {
+                    throw new NotFoundError(AuditableItemStreamService.CLASS_NAME, "streamNotFound", streamId);
+                }
             }
             const result = await this.findEntries(streamNamespaceId, options?.includeDeleted, false, options?.conditions, options?.order, undefined, options?.limit, options?.cursor);
             const list = {
@@ -684,12 +846,12 @@ export class AuditableItemStreamService {
         }
     }
     /**
-     * Remove the verifiable storage for the stream and entries.
-     * @param streamId The id of the stream to remove the storage from.
-     * @returns Nothing.
+     * Remove the proof for the stream and entries.
+     * @param streamId The id of the stream to remove the proof from.
+     * @returns A promise that resolves when the proof has been removed.
      * @throws NotFoundError if the vertex is not found.
      */
-    async removeVerifiable(streamId) {
+    async removeProof(streamId) {
         Guards.stringValue(AuditableItemStreamService.CLASS_NAME, "streamId", streamId);
         const urnParsed = Urn.fromValidString(streamId);
         if (urnParsed.namespaceIdentifier() !== AuditableItemStreamService._NAMESPACE) {
@@ -698,8 +860,9 @@ export class AuditableItemStreamService {
                 id: streamId
             });
         }
+        const streamIdParts = urnParsed.namespaceSpecific(0);
+        await Mutex.lock(streamIdParts, { throwOnTimeout: true, timeoutMs: this._mutexTimeoutMs });
         try {
-            const streamIdParts = urnParsed.namespaceSpecific(0);
             const streamEntity = await this._streamStorage.get(streamIdParts);
             if (Is.empty(streamEntity)) {
                 throw new NotFoundError(AuditableItemStreamService.CLASS_NAME, "streamNotFound", streamIdParts);
@@ -707,9 +870,27 @@ export class AuditableItemStreamService {
             await this.internalRemoveEntries(streamEntity, true);
         }
         catch (error) {
-            throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "removeVerifiableFailed", undefined, error);
+            throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "removeProofFailed", undefined, error);
+        }
+        finally {
+            Mutex.unlock(streamIdParts);
         }
     }
+    /**
+     * Create an immutable proof for the stream entity if the conditions are met.
+     * @param streamEntity The stream entity to create the proof for.
+     * @param immutableInterval The immutable interval for the stream.
+     * @returns The proof id.
+     * @internal
+     */
+    async createStreamProof(streamEntity, immutableInterval) {
+        const streamModel = this.streamEntityToJsonLd(ObjectHelper.pick(streamEntity, AuditableItemStreamService._PROOF_KEYS_STREAM));
+        if (immutableInterval > 0 && Is.stringValue(streamModel.organizationIdentity)) {
+            streamEntity.proofId = await this._immutableProofComponent.create(streamModel);
+            await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.ProofsCreatedStream);
+        }
+        return streamModel.id;
+    }
     /**
      * Map the stream entity to a JSON-LD model.
      * @param streamEntity The stream entity.
@@ -719,9 +900,9 @@ export class AuditableItemStreamService {
     streamEntityToJsonLd(streamEntity) {
         const model = {
             "@context": [
+                SchemaOrgContexts.Context,
                 AuditableItemStreamContexts.Context,
-                AuditableItemStreamContexts.ContextCommon,
-                SchemaOrgContexts.Context
+                AuditableItemStreamContexts.ContextCommon
             ],
             type: AuditableItemStreamTypes.Stream,
             id: `${AuditableItemStreamService._NAMESPACE}:${streamEntity.id}`,
@@ -732,7 +913,9 @@ export class AuditableItemStreamService {
             annotationObject: streamEntity.annotationObject,
             immutableInterval: streamEntity.immutableInterval,
             proofId: streamEntity.proofId,
-            numberOfItems: streamEntity.numberOfItems
+            numberOfItems: streamEntity.numberOfItems,
+            closed: streamEntity.closed,
+            mode: streamEntity.mode
         };
         return model;
     }
@@ -794,8 +977,12 @@ export class AuditableItemStreamService {
             // Create the JSON-LD object we want to use for the proof
             // this is a subset of fixed properties from the stream entry object.
             const streamEntryModel = this.streamEntryEntityToJsonLd(ObjectHelper.pick(entity, AuditableItemStreamService._PROOF_KEYS_STREAM_ENTRY));
-            // Create the proof for the stream object
-            entity.proofId = await this._immutableProofComponent.create(streamEntryModel);
+            // Create the proof for the stream object but only if we have an organization identity,
+            // either from the stream or the context, as this is needed for the proof creation and immutability.
+            if (Is.stringValue(context.organizationIdentity)) {
+                entity.proofId = await this._immutableProofComponent.create(JsonLdHelper.toNodeObject(streamEntryModel));
+                await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.ProofsCreatedEntry, { index: entity.index });
+            }
         }
         await this._streamEntryStorage.set(entity);
         return entity.id;
@@ -805,6 +992,7 @@ export class AuditableItemStreamService {
      * @param streamId The stream id.
      * @param entryId The entry id.
      * @param verifyEntry Should the entry be verified.
+     * @returns The entry entity and optional verification result, or undefined if not found.
      * @internal
      */
     async findEntry(streamId, entryId, verifyEntry) {
@@ -851,18 +1039,19 @@ export class AuditableItemStreamService {
      * @param propertiesToReturn The properties to return.
      * @param limit Limit the number of entities when finding.
      * @param cursor The cursor.
-     * @param contextIds The context ids to perform the operation with.
+     * @returns The stream entries and optional next cursor.
      * @internal
      */
     async findEntries(streamId, includeDeleted, verifyEntries, conditions, sortDirection, propertiesToReturn, limit, cursor) {
         const needToVerify = verifyEntries ?? false;
-        const combinedConditions = [
-            {
+        const combinedConditions = [];
+        if (Is.stringValue(streamId)) {
+            combinedConditions.push({
                 property: "streamId",
                 comparison: ComparisonOperator.Equals,
                 value: streamId
-            }
-        ];
+            });
+        }
         if (Is.stringValue(cursor)) {
             const parts = cursor.split("|");
             if (parts.length > 1) {
@@ -923,14 +1112,15 @@ export class AuditableItemStreamService {
      * Remove the verifiable storage for the stream and entries.
      * @param streamEntity The stream entity.
      * @param removeOnlyProof Should only the proof be removed.
-     * @returns Nothing.
+     * @returns A promise that resolves when the entries have been processed.
      * @internal
      */
     async internalRemoveEntries(streamEntity, removeOnlyProof) {
         if (Is.stringValue(streamEntity.proofId)) {
-            await this._immutableProofComponent.removeVerifiable(streamEntity.proofId);
+            await this._immutableProofComponent.removeNotarization(streamEntity.proofId);
             delete streamEntity.proofId;
             await this._streamStorage.set(streamEntity);
+            await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.ProofsRemovedStream);
         }
         const entryIds = [];
         let entriesResult;
@@ -948,8 +1138,9 @@ export class AuditableItemStreamService {
             for (const streamEntry of entriesResult.entities) {
                 entryIds.push(streamEntry.id);
                 if (Is.stringValue(streamEntry.proofId)) {
-                    await this._immutableProofComponent.removeVerifiable(streamEntry.proofId);
+                    await this._immutableProofComponent.removeNotarization(streamEntry.proofId);
                     delete streamEntry.proofId;
+                    await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.ProofsRemovedEntry);
                     // If we are only removing the proof, we need to set the entry
                     // otherwise the entry is going to be removed later anyway.
                     if (removeOnlyProof) {