@twin.org/auditable-item-stream-service 0.0.3-next.2 → 0.0.3-next.21

package/dist/es/auditableItemStreamService.js

@@ -1,13 +1,15 @@
 // Copyright 2024 IOTA Stiftung.
 // SPDX-License-Identifier: Apache-2.0.
-import { AuditableItemStreamContexts, AuditableItemStreamTopics, AuditableItemStreamTypes } from "@twin.org/auditable-item-stream-models";
-import { ContextIdKeys, ContextIdStore } from "@twin.org/context";
-import { Coerce, ComponentFactory, Converter, GeneralError, Guards, Is, NotFoundError, ObjectHelper, RandomHelper, Urn, Validation } from "@twin.org/core";
-import { JsonLdHelper, JsonLdProcessor } from "@twin.org/data-json-ld";
+import { AuditableItemStreamContexts, AuditableItemStreamDataTypes, AuditableItemStreamMetricIds, AuditableItemStreamMetrics, AuditableItemStreamModes, AuditableItemStreamTopics, AuditableItemStreamTypes } from "@twin.org/auditable-item-stream-models";
+import { ContextIdHelper, ContextIdKeys, ContextIdStore } from "@twin.org/context";
+import { Coerce, ComponentFactory, GeneralError, Guards, Is, Mutex, NotFoundError, ObjectHelper, RandomHelper, Urn, Validation } from "@twin.org/core";
+import { DataTypeHelper } from "@twin.org/data-core";
+import { JsonLdDataTypes, JsonLdHelper, JsonLdProcessor } from "@twin.org/data-json-ld";
 import { ComparisonOperator, LogicalOperator, SortDirection } from "@twin.org/entity";
 import { EntityStorageConnectorFactory } from "@twin.org/entity-storage-models";
-import { ImmutableProofContexts } from "@twin.org/immutable-proof-models";
+import { ImmutableProofContexts, ImmutableProofDataTypes } from "@twin.org/immutable-proof-models";
 import { SchemaOrgContexts, SchemaOrgDataTypes, SchemaOrgTypes } from "@twin.org/standards-schema-org";
+import { MetricHelper } from "@twin.org/telemetry-models";
 /**
  * Class for performing auditable item stream operations.
  */
@@ -68,6 +70,11 @@ export class AuditableItemStreamService {
      * @internal
      */
     _eventBusComponent;
+    /**
+     * The telemetry component.
+     * @internal
+     */
+    _telemetryComponent;
     /**
      * The default interval for the integrity checks.
      * @internal
@@ -84,9 +91,13 @@ export class AuditableItemStreamService {
         if (Is.stringValue(options?.eventBusComponentType)) {
             this._eventBusComponent = ComponentFactory.get(options.eventBusComponentType);
         }
+        this._telemetryComponent = ComponentFactory.getIfExists(options?.telemetryComponentType);
         this._config = options?.config ?? {};
         this._defaultImmutableInterval = this._config.defaultImmutableInterval ?? 10;
         SchemaOrgDataTypes.registerRedirects();
+        AuditableItemStreamDataTypes.registerTypes();
+        JsonLdDataTypes.registerTypes();
+        ImmutableProofDataTypes.registerTypes();
     }
     /**
      * Returns the class name of the component.
@@ -95,75 +106,83 @@ export class AuditableItemStreamService {
     className() {
         return AuditableItemStreamService.CLASS_NAME;
     }
+    /**
+     * Register all AIS metrics with the telemetry component.
+     */
+    async start() {
+        if (Is.undefined(this._telemetryComponent)) {
+            return;
+        }
+        await MetricHelper.createMetrics(this._telemetryComponent, AuditableItemStreamMetrics);
+    }
     /**
      * Create a new stream.
      * @param stream The stream to create.
-     * @param stream.annotationObject The object for the stream as JSON-LD.
-     * @param stream.entries Entries to store in the stream.
-     * @param options Options for creating the stream.
-     * @param options.immutableInterval After how many entries do we add immutable checks, defaults to service configured value.
-     * A value of 0 will disable integrity checks, 1 will be every item, or any other integer for an interval.
      * @returns The id of the new stream item.
      */
-    async create(stream, options) {
+    async create(stream) {
         Guards.object(AuditableItemStreamService.CLASS_NAME, "stream", stream);
         const contextIds = await ContextIdStore.getContextIds();
+        ContextIdHelper.guard(contextIds, ContextIdKeys.Organization);
         try {
+            const ownerOrganizationId = contextIds[ContextIdKeys.UserOrganization] ?? contextIds[ContextIdKeys.Organization];
+            const id = RandomHelper.generateUuidV7("compact");
+            const schemaValidationFailures = [];
+            await DataTypeHelper.validate("stream", `${AuditableItemStreamContexts.Namespace}${AuditableItemStreamTypes.Stream}Base`, stream, schemaValidationFailures);
+            Validation.asValidationError(AuditableItemStreamService.CLASS_NAME, "stream", schemaValidationFailures);
+            if (stream.closed && !Is.arrayValue(stream.entries?.[SchemaOrgTypes.ItemListElement])) {
+                throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "closedRequiresEntries");
+            }
             if (Is.object(stream.annotationObject)) {
                 const validationFailures = [];
                 await JsonLdHelper.validate(stream.annotationObject, validationFailures);
                 Validation.asValidationError(AuditableItemStreamService.CLASS_NAME, "stream.annotationObject", validationFailures);
             }
-            const id = Converter.bytesToHex(RandomHelper.generate(32), false);
             const context = {
                 now: new Date(Date.now()).toISOString(),
                 contextIds,
                 indexCounter: 0,
-                immutableInterval: options?.immutableInterval ?? this._defaultImmutableInterval
+                immutableInterval: stream?.immutableInterval ?? this._defaultImmutableInterval,
+                organizationIdentity: ownerOrganizationId
             };
             const streamEntity = {
                 id,
-                organizationIdentity: contextIds?.[ContextIdKeys.Organization],
+                organizationIdentity: ownerOrganizationId,
                 userIdentity: contextIds?.[ContextIdKeys.User],
                 dateCreated: context.now,
                 immutableInterval: context.immutableInterval,
-                indexCounter: 0,
-                proofId: ""
+                closed: stream.closed,
+                mode: stream.mode,
+                numberOfItems: 0
             };
-            // Create the JSON-LD object we want to use for the proof
-            // this is a subset of fixed properties from the stream object.
-            const streamModel = this.streamEntityToJsonLd(ObjectHelper.pick(streamEntity, AuditableItemStreamService._PROOF_KEYS_STREAM));
-            // Create the proof for the stream object
-            streamEntity.proofId = await this._immutableProofComponent.create(streamModel);
-            if (Is.arrayValue(stream.entries)) {
-                for (const entry of stream.entries) {
+            const streamUrn = await this.createStreamProof(streamEntity, context.immutableInterval);
+            if (Is.arrayValue(stream.entries?.[SchemaOrgTypes.ItemListElement])) {
+                for (const entry of stream.entries[SchemaOrgTypes.ItemListElement]) {
                     await this.setEntry(context, id, entry);
                 }
             }
             // Add these dynamic properties to the stream object after the proof has been created.
             streamEntity.dateModified = context.now;
             streamEntity.annotationObject = stream.annotationObject;
-            streamEntity.indexCounter = context.indexCounter;
+            streamEntity.numberOfItems = context.indexCounter;
             await this._streamStorage.set(streamEntity);
-            await this._eventBusComponent?.publish(AuditableItemStreamTopics.StreamCreated, { id: streamModel.id });
-            return streamModel.id;
+            await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.StreamsCreated, {
+                mode: streamEntity.mode ?? AuditableItemStreamModes.Default,
+                immutableInterval: context.immutableInterval
+            });
+            await this._eventBusComponent?.publish(AuditableItemStreamTopics.StreamCreated, { id: streamUrn });
+            return streamUrn;
         }
         catch (error) {
             throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "createFailed", undefined, error);
         }
     }
     /**
-     * Get a stream header without the entries.
-     * @param id The id of the stream to get.
-     * @param options Additional options for the get operation.
-     * @param options.includeEntries Whether to include the entries, defaults to false.
-     * @param options.includeDeleted Whether to include deleted entries, defaults to false.
-     * @param options.verifyStream Should the stream be verified, defaults to false.
-     * @param options.verifyEntries Should the entries be verified, defaults to false.
-     * @returns The stream and entries if found.
-     * @throws NotFoundError if the stream is not found
+     * Close a stream.
+     * @param id The id of the stream to close.
+     * @returns Nothing.
      */
-    async get(id, options) {
+    async close(id) {
         Guards.stringValue(AuditableItemStreamService.CLASS_NAME, "id", id);
         const urnParsed = Urn.fromValidString(id);
         if (urnParsed.namespaceIdentifier() !== AuditableItemStreamService._NAMESPACE) {
@@ -172,38 +191,31 @@ export class AuditableItemStreamService {
                 id
             });
         }
+        const streamId = urnParsed.namespaceSpecific(0);
+        await Mutex.lock(streamId, { throwOnTimeout: true });
         try {
-            const streamId = urnParsed.namespaceSpecific(0);
             const streamEntity = await this._streamStorage.get(streamId);
             if (Is.empty(streamEntity)) {
                 throw new NotFoundError(AuditableItemStreamService.CLASS_NAME, "streamNotFound", id);
             }
-            const verifyStream = options?.verifyStream ?? false;
-            const verifyEntries = options?.verifyEntries ?? false;
-            const streamModel = this.streamEntityToJsonLd(streamEntity);
-            if (options?.includeEntries) {
-                const result = await this.findEntries(streamId, options?.includeDeleted, verifyEntries);
-                streamModel.entries = result.entries;
-                streamModel.cursor = result.cursor;
-            }
-            if (verifyStream && Is.stringValue(streamEntity.proofId)) {
-                streamModel.verification = await this._immutableProofComponent.verify(streamEntity.proofId);
-            }
-            if (verifyStream || verifyEntries) {
-                streamModel["@context"].push(ImmutableProofContexts.ContextRoot);
+            if (!streamEntity.closed) {
+                streamEntity.closed = true;
+                streamEntity.dateModified = new Date(Date.now()).toISOString();
+                await this._streamStorage.set(streamEntity);
+                await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.StreamsClosed);
+                await this._eventBusComponent?.publish(AuditableItemStreamTopics.StreamUpdated, { id });
             }
-            const result = await JsonLdProcessor.compact(streamModel, streamModel["@context"]);
-            return result;
         }
         catch (error) {
-            throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "getFailed", undefined, error);
+            throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "closeFailed", undefined, error);
+        }
+        finally {
+            Mutex.unlock(streamId);
         }
     }
     /**
      * Update a stream.
-     * @param stream The stream to update.
-     * @param stream.id The id of the stream to update.
-     * @param stream.annotationObject The object for the stream as JSON-LD.
+     * @param stream The stream to update, does not update entries.
      * @returns Nothing.
      */
     async update(stream) {
@@ -216,8 +228,12 @@ export class AuditableItemStreamService {
                 id: stream.id
             });
         }
+        const streamId = urnParsed.namespaceSpecific(0);
+        await Mutex.lock(streamId, { throwOnTimeout: true });
         try {
-            const streamId = urnParsed.namespaceSpecific(0);
+            const schemaValidationFailures = [];
+            await DataTypeHelper.validate("stream", `${AuditableItemStreamContexts.Namespace}${AuditableItemStreamTypes.Stream}`, stream, schemaValidationFailures);
+            Validation.asValidationError(AuditableItemStreamService.CLASS_NAME, "stream", schemaValidationFailures);
             const streamEntity = await this._streamStorage.get(streamId);
             if (Is.empty(streamEntity)) {
                 throw new NotFoundError(AuditableItemStreamService.CLASS_NAME, "streamNotFound", stream.id);
@@ -227,16 +243,94 @@ export class AuditableItemStreamService {
                 await JsonLdHelper.validate(stream.annotationObject, validationFailures);
                 Validation.asValidationError(AuditableItemStreamService.CLASS_NAME, "stream.annotationObject", validationFailures);
             }
+            let changed = false;
             if (!ObjectHelper.equal(streamEntity.annotationObject, stream.annotationObject, false)) {
                 streamEntity.annotationObject = stream.annotationObject;
+                changed = true;
+            }
+            const contextIds = await ContextIdStore.getContextIds();
+            const ownerOrganizationId = contextIds?.[ContextIdKeys.UserOrganization] ?? contextIds?.[ContextIdKeys.Organization];
+            if (!Is.stringValue(streamEntity.organizationIdentity) &&
+                Is.stringValue(ownerOrganizationId)) {
+                streamEntity.organizationIdentity = ownerOrganizationId;
+                changed = true;
+            }
+            if (!Is.stringValue(streamEntity.proofId) &&
+                Is.stringValue(streamEntity.organizationIdentity)) {
+                await this.createStreamProof(streamEntity, streamEntity.immutableInterval ?? this._defaultImmutableInterval);
+                if (Is.stringValue(streamEntity.proofId)) {
+                    changed = true;
+                }
+            }
+            if (changed) {
                 streamEntity.dateModified = new Date(Date.now()).toISOString();
                 await this._streamStorage.set(streamEntity);
+                await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.StreamsUpdated);
                 await this._eventBusComponent?.publish(AuditableItemStreamTopics.StreamUpdated, { id: stream.id });
             }
         }
         catch (error) {
             throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "updatingFailed", undefined, error);
         }
+        finally {
+            Mutex.unlock(streamId);
+        }
+    }
+    /**
+     * Get a stream header without the entries.
+     * @param id The id of the stream to get.
+     * @param cursor Cursor to use for next chunk of entries.
+     * @param limit Limit the number of entries to return, only applicable if includeEntries is true.
+     * @param options Additional options for the get operation.
+     * @param options.includeEntries Whether to include the entries, defaults to false.
+     * @param options.includeDeleted Whether to include deleted entries, defaults to false.
+     * @param options.verifyStream Should the stream be verified, defaults to false.
+     * @param options.verifyEntries Should the entries be verified, defaults to false.
+     * @returns The stream and entries if found.
+     * @throws NotFoundError if the stream is not found
+     */
+    async get(id, cursor, limit, options) {
+        Guards.stringValue(AuditableItemStreamService.CLASS_NAME, "id", id);
+        const urnParsed = Urn.fromValidString(id);
+        if (urnParsed.namespaceIdentifier() !== AuditableItemStreamService._NAMESPACE) {
+            throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "namespaceMismatch", {
+                namespace: AuditableItemStreamService._NAMESPACE,
+                id
+            });
+        }
+        try {
+            const streamId = urnParsed.namespaceSpecific(0);
+            const streamEntity = await this._streamStorage.get(streamId);
+            if (Is.empty(streamEntity)) {
+                throw new NotFoundError(AuditableItemStreamService.CLASS_NAME, "streamNotFound", id);
+            }
+            const verifyStream = options?.verifyStream ?? false;
+            const verifyEntries = options?.verifyEntries ?? false;
+            const streamModel = this.streamEntityToJsonLd(streamEntity);
+            let returnCursor;
+            if (options?.includeEntries) {
+                const result = await this.findEntries(streamId, options?.includeDeleted, verifyEntries, undefined, undefined, undefined, limit, cursor);
+                streamModel.entries = {
+                    type: SchemaOrgTypes.ItemList,
+                    [SchemaOrgTypes.ItemListElement]: result.entries
+                };
+                returnCursor = result.cursor;
+            }
+            if (verifyStream && Is.stringValue(streamEntity.proofId)) {
+                streamModel.verification = await this._immutableProofComponent.verify(streamEntity.proofId);
+            }
+            if (verifyStream || verifyEntries) {
+                streamModel["@context"].push(ImmutableProofContexts.Context);
+            }
+            const result = await JsonLdProcessor.compact(streamModel, streamModel["@context"]);
+            return {
+                stream: result,
+                cursor: returnCursor
+            };
+        }
+        catch (error) {
+            throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "getFailed", undefined, error);
+        }
     }
     /**
      * Delete the stream.
@@ -252,19 +346,24 @@ export class AuditableItemStreamService {
                 id
             });
         }
+        const streamId = urnParsed.namespaceSpecific(0);
+        await Mutex.lock(streamId, { throwOnTimeout: true });
         try {
-            const streamId = urnParsed.namespaceSpecific(0);
             const streamEntity = await this._streamStorage.get(streamId);
             if (Is.empty(streamEntity)) {
                 throw new NotFoundError(AuditableItemStreamService.CLASS_NAME, "streamNotFound", id);
             }
             await this.internalRemoveEntries(streamEntity, false);
             await this._streamStorage.remove(streamEntity.id);
+            await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.StreamsDeleted);
             await this._eventBusComponent?.publish(AuditableItemStreamTopics.StreamDeleted, { id });
         }
         catch (error) {
             throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "removingFailed", undefined, error);
         }
+        finally {
+            Mutex.unlock(streamId);
+        }
     }
     /**
      * Query all the streams, will not return entries.
@@ -305,16 +404,18 @@ export class AuditableItemStreamService {
             ], propertiesToReturn, cursor, limit);
             const list = {
                 "@context": [
-                    SchemaOrgContexts.ContextRoot,
-                    AuditableItemStreamContexts.ContextRoot,
-                    AuditableItemStreamContexts.ContextRootCommon
+                    SchemaOrgContexts.Context,
+                    AuditableItemStreamContexts.Context,
+                    AuditableItemStreamContexts.ContextCommon
                 ],
                 type: [SchemaOrgTypes.ItemList, AuditableItemStreamTypes.StreamList],
-                [SchemaOrgTypes.ItemListElement]: results.entities.map(e => this.streamEntityToJsonLd(e)),
-                [SchemaOrgTypes.NextItem]: results.cursor
+                [SchemaOrgTypes.ItemListElement]: results.entities.map(e => this.streamEntityToJsonLd(e))
             };
             const result = await JsonLdProcessor.compact(list, list["@context"]);
-            return result;
+            return {
+                entries: result,
+                cursor: results.cursor
+            };
         }
         catch (error) {
             throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "queryingFailed", undefined, error);
@@ -336,24 +437,36 @@ export class AuditableItemStreamService {
                 id: streamId
             });
         }
+        const streamIdParts = urnParsed.namespaceSpecific(0);
+        await Mutex.lock(streamIdParts, { throwOnTimeout: true });
         try {
-            const streamIdParts = urnParsed.namespaceSpecific(0);
             const streamEntity = await this._streamStorage.get(streamIdParts);
             if (Is.empty(streamEntity)) {
                 throw new NotFoundError(AuditableItemStreamService.CLASS_NAME, "streamNotFound", streamIdParts);
             }
+            if (streamEntity.closed) {
+                await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.ClosedStreamRejections, { operation: "createEntry" });
+                throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "streamClosed", {
+                    id: streamId
+                });
+            }
             const context = {
                 now: new Date(Date.now()).toISOString(),
                 contextIds,
-                indexCounter: streamEntity.indexCounter,
-                immutableInterval: streamEntity.immutableInterval
+                indexCounter: streamEntity.numberOfItems,
+                immutableInterval: streamEntity.immutableInterval,
+                organizationIdentity: streamEntity.organizationIdentity ?? contextIds?.[ContextIdKeys.Organization]
             };
             const createdId = await this.setEntry(context, streamEntity.id, {
                 entryObject
             });
             streamEntity.dateModified = context.now;
-            streamEntity.indexCounter = context.indexCounter;
+            streamEntity.numberOfItems = context.indexCounter;
             await this._streamStorage.set(streamEntity);
+            await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.EntriesCreated, {
+                hasProof: context.immutableInterval > 0 &&
+                    (context.indexCounter - 1) % context.immutableInterval === 0
+            });
             const fullId = new Urn(AuditableItemStreamService._NAMESPACE, [
                 streamEntity.id,
                 createdId
@@ -364,6 +477,9 @@ export class AuditableItemStreamService {
         catch (error) {
             throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "creatingEntryFailed", undefined, error);
         }
+        finally {
+            Mutex.unlock(streamIdParts);
+        }
     }
     /**
      * Get the entry from the stream.
@@ -405,7 +521,7 @@ export class AuditableItemStreamService {
             }
             const entry = this.streamEntryEntityToJsonLd(result.entity);
             if (verifyEntry) {
-                entry["@context"].push(ImmutableProofContexts.ContextRoot);
+                entry["@context"] = JsonLdProcessor.combineContexts(entry["@context"], ImmutableProofContexts.Context);
                 entry.verification = result.verification;
             }
             const result2 = await JsonLdProcessor.compact(entry, entry["@context"]);
@@ -466,7 +582,6 @@ export class AuditableItemStreamService {
     async updateEntry(streamId, entryId, entryObject) {
         Guards.stringValue(AuditableItemStreamService.CLASS_NAME, "streamId", streamId);
         Guards.stringValue(AuditableItemStreamService.CLASS_NAME, "entryId", entryId);
-        const contextIds = await ContextIdStore.getContextIds();
         const urnParsed = Urn.fromValidString(streamId);
         if (urnParsed.namespaceIdentifier() !== AuditableItemStreamService._NAMESPACE) {
             throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "namespaceMismatch", {
@@ -481,8 +596,9 @@ export class AuditableItemStreamService {
                 id: entryId
             });
         }
+        const streamNamespaceId = urnParsed.namespaceSpecific(0);
+        await Mutex.lock(streamNamespaceId, { throwOnTimeout: true });
         try {
-            const streamNamespaceId = urnParsed.namespaceMethod();
             const streamEntryNamespaceId = urnParsedEntry.namespaceMethod();
             if (streamNamespaceId !== streamEntryNamespaceId) {
                 throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "namespaceMismatch", {
@@ -494,29 +610,48 @@ export class AuditableItemStreamService {
             if (Is.empty(streamEntity)) {
                 throw new NotFoundError(AuditableItemStreamService.CLASS_NAME, "streamNotFound", streamId);
             }
+            if (streamEntity.closed) {
+                await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.ClosedStreamRejections, { operation: "updateEntry" });
+                throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "streamClosed", {
+                    id: streamId
+                });
+            }
+            if (streamEntity.mode === AuditableItemStreamModes.AppendOnly) {
+                await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.AppendOnlyRejections, { operation: "updateEntry" });
+                throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "appendOnlyNoEntryUpdates", {
+                    id: streamId
+                });
+            }
             const entryNamespaceId = urnParsedEntry.namespaceSpecific(1);
             const existing = await this.findEntry(streamEntity.id, entryNamespaceId);
             if (Is.empty(existing)) {
                 throw new NotFoundError(AuditableItemStreamService.CLASS_NAME, "streamEntryNotFound", entryId);
             }
+            const contextIds = await ContextIdStore.getContextIds();
+            const ownerOrganizationId = contextIds?.[ContextIdKeys.UserOrganization] ?? contextIds?.[ContextIdKeys.Organization];
             const context = {
                 now: new Date(Date.now()).toISOString(),
                 contextIds,
-                indexCounter: streamEntity.indexCounter,
-                immutableInterval: streamEntity.immutableInterval
+                indexCounter: streamEntity.numberOfItems,
+                immutableInterval: streamEntity.immutableInterval,
+                organizationIdentity: streamEntity.organizationIdentity ?? ownerOrganizationId
             };
             await this.setEntry(context, streamEntity.id, {
                 ...existing.entity,
                 entryObject
             });
             streamEntity.dateModified = context.now;
-            streamEntity.indexCounter = context.indexCounter;
+            streamEntity.numberOfItems = context.indexCounter;
             await this._streamStorage.set(streamEntity);
+            await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.EntriesUpdated);
             await this._eventBusComponent?.publish(AuditableItemStreamTopics.StreamEntryUpdated, { id: streamId, entryId });
         }
         catch (error) {
             throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "updatingEntryFailed", undefined, error);
         }
+        finally {
+            Mutex.unlock(streamNamespaceId);
+        }
     }
     /**
      * Delete from the stream.
@@ -527,7 +662,6 @@ export class AuditableItemStreamService {
     async removeEntry(streamId, entryId) {
         Guards.stringValue(AuditableItemStreamService.CLASS_NAME, "streamId", streamId);
         Guards.stringValue(AuditableItemStreamService.CLASS_NAME, "entryId", entryId);
-        const contextIds = await ContextIdStore.getContextIds();
         const urnParsed = Urn.fromValidString(streamId);
         if (urnParsed.namespaceIdentifier() !== AuditableItemStreamService._NAMESPACE) {
             throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "namespaceMismatch", {
@@ -542,8 +676,9 @@ export class AuditableItemStreamService {
                 id: entryId
             });
         }
+        const streamNamespaceId = urnParsed.namespaceSpecific(0);
+        await Mutex.lock(streamNamespaceId, { throwOnTimeout: true });
         try {
-            const streamNamespaceId = urnParsed.namespaceMethod();
             const streamEntryNamespaceId = urnParsedEntry.namespaceMethod();
             if (streamNamespaceId !== streamEntryNamespaceId) {
                 throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "namespaceMismatch", {
@@ -555,35 +690,48 @@ export class AuditableItemStreamService {
             if (Is.empty(streamEntity)) {
                 throw new NotFoundError(AuditableItemStreamService.CLASS_NAME, "streamNotFound", streamId);
             }
+            if (streamEntity.mode === AuditableItemStreamModes.AppendOnly) {
+                await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.AppendOnlyRejections, { operation: "removeEntry" });
+                throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "appendOnlyNoEntryRemovals", {
+                    id: streamId
+                });
+            }
             const entryNamespaceId = urnParsedEntry.namespaceSpecific(1);
             const result = await this.findEntry(streamNamespaceId, entryNamespaceId);
             if (Is.empty(result)) {
                 throw new NotFoundError(AuditableItemStreamService.CLASS_NAME, "streamEntryNotFound", entryId);
             }
             if (Is.empty(result.entity.dateDeleted)) {
+                const contextIds = await ContextIdStore.getContextIds();
+                const ownerOrganizationId = contextIds?.[ContextIdKeys.UserOrganization] ?? contextIds?.[ContextIdKeys.Organization];
                 const context = {
                     now: new Date(Date.now()).toISOString(),
                     contextIds,
-                    indexCounter: streamEntity.indexCounter,
-                    immutableInterval: streamEntity.immutableInterval
+                    indexCounter: streamEntity.numberOfItems,
+                    immutableInterval: streamEntity.immutableInterval,
+                    organizationIdentity: streamEntity.organizationIdentity ?? ownerOrganizationId
                 };
                 await this.setEntry(context, streamEntity.id, {
                     ...result.entity,
                     dateDeleted: context.now
                 });
                 streamEntity.dateModified = context.now;
-                streamEntity.indexCounter = context.indexCounter;
+                streamEntity.numberOfItems = context.indexCounter;
                 await this._streamStorage.set(streamEntity);
+                await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.EntriesDeleted);
                 await this._eventBusComponent?.publish(AuditableItemStreamTopics.StreamEntryDeleted, { id: streamId, entryId });
             }
         }
         catch (error) {
             throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "removingEntryFailed", undefined, error);
         }
+        finally {
+            Mutex.unlock(streamNamespaceId);
+        }
     }
     /**
      * Get the entries for the stream.
-     * @param streamId The id of the stream to get.
+     * @param streamId The id of the stream to get, if undefined returns all matching entries.
      * @param options Additional options for the get operation.
      * @param options.conditions The conditions to filter the stream.
      * @param options.includeDeleted Whether to include deleted entries, defaults to false.
@@ -595,37 +743,44 @@ export class AuditableItemStreamService {
      * @throws NotFoundError if the stream is not found.
      */
     async getEntries(streamId, options) {
-        Guards.stringValue(AuditableItemStreamService.CLASS_NAME, "streamId", streamId);
-        const urnParsed = Urn.fromValidString(streamId);
-        if (urnParsed.namespaceIdentifier() !== AuditableItemStreamService._NAMESPACE) {
-            throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "namespaceMismatch", {
-                namespace: AuditableItemStreamService._NAMESPACE,
-                id: streamId
-            });
+        let streamNamespaceId;
+        if (!Is.empty(streamId)) {
+            Guards.stringValue(AuditableItemStreamService.CLASS_NAME, "streamId", streamId);
+            const urnParsed = Urn.fromValidString(streamId);
+            if (urnParsed.namespaceIdentifier() !== AuditableItemStreamService._NAMESPACE) {
+                throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "namespaceMismatch", {
+                    namespace: AuditableItemStreamService._NAMESPACE,
+                    id: streamId
+                });
+            }
+            streamNamespaceId = urnParsed.namespaceSpecific(0);
         }
         try {
-            const streamNamespaceId = urnParsed.namespaceSpecific(0);
-            const streamEntity = await this._streamStorage.get(streamNamespaceId);
-            if (Is.empty(streamEntity)) {
-                throw new NotFoundError(AuditableItemStreamService.CLASS_NAME, "streamNotFound", streamId);
+            if (Is.stringValue(streamNamespaceId)) {
+                const streamEntity = await this._streamStorage.get(streamNamespaceId);
+                if (Is.empty(streamEntity)) {
+                    throw new NotFoundError(AuditableItemStreamService.CLASS_NAME, "streamNotFound", streamId);
+                }
             }
             const verifyEntries = options?.verifyEntries ?? false;
             const result = await this.findEntries(streamNamespaceId, options?.includeDeleted, verifyEntries, options?.conditions, options?.order, undefined, options?.limit, options?.cursor);
             const list = {
                 "@context": [
-                    SchemaOrgContexts.ContextRoot,
-                    AuditableItemStreamContexts.ContextRoot,
-                    AuditableItemStreamContexts.ContextRootCommon
+                    SchemaOrgContexts.Context,
+                    AuditableItemStreamContexts.Context,
+                    AuditableItemStreamContexts.ContextCommon
                 ],
                 type: [SchemaOrgTypes.ItemList, AuditableItemStreamTypes.StreamEntryList],
-                [SchemaOrgTypes.ItemListElement]: result.entries,
-                [SchemaOrgTypes.NextItem]: result.cursor
+                [SchemaOrgTypes.ItemListElement]: result.entries
             };
             if (verifyEntries) {
-                list["@context"].push(ImmutableProofContexts.ContextRoot);
+                list["@context"].push(ImmutableProofContexts.Context);
             }
             const result2 = await JsonLdProcessor.compact(list, list["@context"]);
-            return result2;
+            return {
+                entries: result2,
+                cursor: result.cursor
+            };
         }
         catch (error) {
             throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "gettingEntriesFailed", undefined, error);
@@ -633,7 +788,7 @@ export class AuditableItemStreamService {
     }
     /**
      * Get the entry objects for the stream.
-     * @param streamId The id of the stream to get.
+     * @param streamId The id of the stream to get, if undefined returns all matching entries.
      * @param options Additional options for the get operation.
      * @param options.conditions The conditions to filter the stream.
      * @param options.includeDeleted Whether to include deleted entries, defaults to false.
@@ -644,45 +799,52 @@ export class AuditableItemStreamService {
      * @throws NotFoundError if the stream is not found.
      */
     async getEntryObjects(streamId, options) {
-        Guards.stringValue(AuditableItemStreamService.CLASS_NAME, "streamId", streamId);
-        const urnParsed = Urn.fromValidString(streamId);
-        if (urnParsed.namespaceIdentifier() !== AuditableItemStreamService._NAMESPACE) {
-            throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "namespaceMismatch", {
-                namespace: AuditableItemStreamService._NAMESPACE,
-                id: streamId
-            });
+        let streamNamespaceId;
+        if (!Is.empty(streamId)) {
+            Guards.stringValue(AuditableItemStreamService.CLASS_NAME, "streamId", streamId);
+            const urnParsed = Urn.fromValidString(streamId);
+            if (urnParsed.namespaceIdentifier() !== AuditableItemStreamService._NAMESPACE) {
+                throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "namespaceMismatch", {
+                    namespace: AuditableItemStreamService._NAMESPACE,
+                    id: streamId
+                });
+            }
+            streamNamespaceId = urnParsed.namespaceSpecific(0);
         }
         try {
-            const streamNamespaceId = urnParsed.namespaceSpecific(0);
-            const streamEntity = await this._streamStorage.get(streamNamespaceId);
-            if (Is.empty(streamEntity)) {
-                throw new NotFoundError(AuditableItemStreamService.CLASS_NAME, "streamNotFound", streamId);
+            if (Is.stringValue(streamNamespaceId)) {
+                const streamEntity = await this._streamStorage.get(streamNamespaceId);
+                if (Is.empty(streamEntity)) {
+                    throw new NotFoundError(AuditableItemStreamService.CLASS_NAME, "streamNotFound", streamId);
+                }
             }
             const result = await this.findEntries(streamNamespaceId, options?.includeDeleted, false, options?.conditions, options?.order, undefined, options?.limit, options?.cursor);
             const list = {
                 "@context": [
-                    SchemaOrgContexts.ContextRoot,
-                    AuditableItemStreamContexts.ContextRoot,
-                    AuditableItemStreamContexts.ContextRootCommon
+                    SchemaOrgContexts.Context,
+                    AuditableItemStreamContexts.Context,
+                    AuditableItemStreamContexts.ContextCommon
                 ],
                 type: [SchemaOrgTypes.ItemList, AuditableItemStreamTypes.StreamEntryObjectList],
-                [SchemaOrgTypes.ItemListElement]: result.entries.map(m => m.entryObject),
-                [SchemaOrgTypes.NextItem]: result.cursor
+                [SchemaOrgTypes.ItemListElement]: result.entries.map(m => m.entryObject)
             };
             const result2 = await JsonLdProcessor.compact(list, list["@context"]);
-            return result2;
+            return {
+                entries: result2,
+                cursor: result.cursor
+            };
         }
         catch (error) {
             throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "gettingEntryObjectsFailed", undefined, error);
         }
     }
     /**
-     * Remove the verifiable storage for the stream and entries.
-     * @param streamId The id of the stream to remove the storage from.
+     * Remove the proof for the stream and entries.
+     * @param streamId The id of the stream to remove the proof from.
      * @returns Nothing.
      * @throws NotFoundError if the vertex is not found.
      */
-    async removeVerifiable(streamId) {
+    async removeProof(streamId) {
         Guards.stringValue(AuditableItemStreamService.CLASS_NAME, "streamId", streamId);
         const urnParsed = Urn.fromValidString(streamId);
         if (urnParsed.namespaceIdentifier() !== AuditableItemStreamService._NAMESPACE) {
@@ -691,8 +853,9 @@ export class AuditableItemStreamService {
                 id: streamId
             });
         }
+        const streamIdParts = urnParsed.namespaceSpecific(0);
+        await Mutex.lock(streamIdParts, { throwOnTimeout: true });
         try {
-            const streamIdParts = urnParsed.namespaceSpecific(0);
             const streamEntity = await this._streamStorage.get(streamIdParts);
             if (Is.empty(streamEntity)) {
                 throw new NotFoundError(AuditableItemStreamService.CLASS_NAME, "streamNotFound", streamIdParts);
@@ -700,9 +863,27 @@ export class AuditableItemStreamService {
             await this.internalRemoveEntries(streamEntity, true);
         }
         catch (error) {
-            throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "removeVerifiableFailed", undefined, error);
+            throw new GeneralError(AuditableItemStreamService.CLASS_NAME, "removeProofFailed", undefined, error);
+        }
+        finally {
+            Mutex.unlock(streamIdParts);
         }
     }
+    /**
+     * Create an immutable proof for the stream entity if the conditions are met.
+     * @param streamEntity The stream entity to create the proof for.
+     * @param immutableInterval The immutable interval for the stream.
+     * @returns The proof id.
+     * @internal
+     */
+    async createStreamProof(streamEntity, immutableInterval) {
+        const streamModel = this.streamEntityToJsonLd(ObjectHelper.pick(streamEntity, AuditableItemStreamService._PROOF_KEYS_STREAM));
+        if (immutableInterval > 0 && Is.stringValue(streamModel.organizationIdentity)) {
+            streamEntity.proofId = await this._immutableProofComponent.create(streamModel);
+            await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.ProofsCreatedStream);
+        }
+        return streamModel.id;
+    }
     /**
      * Map the stream entity to a JSON-LD model.
      * @param streamEntity The stream entity.
@@ -712,9 +893,9 @@ export class AuditableItemStreamService {
     streamEntityToJsonLd(streamEntity) {
         const model = {
             "@context": [
-                AuditableItemStreamContexts.ContextRoot,
-                AuditableItemStreamContexts.ContextRootCommon,
-                SchemaOrgContexts.ContextRoot
+                SchemaOrgContexts.Context,
+                AuditableItemStreamContexts.Context,
+                AuditableItemStreamContexts.ContextCommon
             ],
             type: AuditableItemStreamTypes.Stream,
             id: `${AuditableItemStreamService._NAMESPACE}:${streamEntity.id}`,
@@ -724,7 +905,10 @@ export class AuditableItemStreamService {
             userIdentity: streamEntity.userIdentity,
             annotationObject: streamEntity.annotationObject,
             immutableInterval: streamEntity.immutableInterval,
-            proofId: streamEntity.proofId
+            proofId: streamEntity.proofId,
+            numberOfItems: streamEntity.numberOfItems,
+            closed: streamEntity.closed,
+            mode: streamEntity.mode
         };
         return model;
     }
@@ -737,9 +921,9 @@ export class AuditableItemStreamService {
     streamEntryEntityToJsonLd(streamEntryEntity) {
         const streamEntryModel = {
             "@context": [
-                AuditableItemStreamContexts.ContextRoot,
-                AuditableItemStreamContexts.ContextRootCommon,
-                SchemaOrgContexts.ContextRoot
+                AuditableItemStreamContexts.Context,
+                AuditableItemStreamContexts.ContextCommon,
+                SchemaOrgContexts.Context
             ],
             type: AuditableItemStreamTypes.StreamEntry,
             id: `${AuditableItemStreamService._NAMESPACE}:${streamEntryEntity.streamId}:${streamEntryEntity.id}`,
@@ -770,7 +954,7 @@ export class AuditableItemStreamService {
             Validation.asValidationError(AuditableItemStreamService.CLASS_NAME, "entry.entryObject", validationFailures);
         }
         const entity = {
-            id: entry.id ?? Converter.bytesToHex(RandomHelper.generate(32), false),
+            id: entry.id ?? RandomHelper.generateUuidV7("compact"),
             streamId,
             dateCreated: entry.dateCreated ?? context.now,
             dateDeleted: entry.dateDeleted,
@@ -786,8 +970,12 @@ export class AuditableItemStreamService {
             // Create the JSON-LD object we want to use for the proof
             // this is a subset of fixed properties from the stream entry object.
             const streamEntryModel = this.streamEntryEntityToJsonLd(ObjectHelper.pick(entity, AuditableItemStreamService._PROOF_KEYS_STREAM_ENTRY));
-            // Create the proof for the stream object
-            entity.proofId = await this._immutableProofComponent.create(streamEntryModel);
+            // Create the proof for the stream object but only if we have an organization identity,
+            // either from the stream or the context, as this is needed for the proof creation and immutability.
+            if (Is.stringValue(context.organizationIdentity)) {
+                entity.proofId = await this._immutableProofComponent.create(JsonLdHelper.toNodeObject(streamEntryModel));
+                await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.ProofsCreatedEntry, { index: entity.index });
+            }
         }
         await this._streamEntryStorage.set(entity);
         return entity.id;
@@ -797,6 +985,7 @@ export class AuditableItemStreamService {
      * @param streamId The stream id.
      * @param entryId The entry id.
      * @param verifyEntry Should the entry be verified.
+     * @returns The entry entity and optional verification result, or undefined if not found.
      * @internal
      */
     async findEntry(streamId, entryId, verifyEntry) {
@@ -843,18 +1032,19 @@ export class AuditableItemStreamService {
      * @param propertiesToReturn The properties to return.
      * @param limit Limit the number of entities when finding.
      * @param cursor The cursor.
-     * @param contextIds The context ids to perform the operation with.
+     * @returns The stream entries and optional next cursor.
      * @internal
      */
     async findEntries(streamId, includeDeleted, verifyEntries, conditions, sortDirection, propertiesToReturn, limit, cursor) {
         const needToVerify = verifyEntries ?? false;
-        const combinedConditions = [
-            {
+        const combinedConditions = [];
+        if (Is.stringValue(streamId)) {
+            combinedConditions.push({
                 property: "streamId",
                 comparison: ComparisonOperator.Equals,
                 value: streamId
-            }
-        ];
+            });
+        }
         if (Is.stringValue(cursor)) {
             const parts = cursor.split("|");
             if (parts.length > 1) {
@@ -920,9 +1110,10 @@ export class AuditableItemStreamService {
      */
     async internalRemoveEntries(streamEntity, removeOnlyProof) {
         if (Is.stringValue(streamEntity.proofId)) {
-            await this._immutableProofComponent.removeVerifiable(streamEntity.proofId);
+            await this._immutableProofComponent.removeNotarization(streamEntity.proofId);
             delete streamEntity.proofId;
             await this._streamStorage.set(streamEntity);
+            await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.ProofsRemovedStream);
         }
         const entryIds = [];
         let entriesResult;
@@ -940,8 +1131,9 @@ export class AuditableItemStreamService {
             for (const streamEntry of entriesResult.entities) {
                 entryIds.push(streamEntry.id);
                 if (Is.stringValue(streamEntry.proofId)) {
-                    await this._immutableProofComponent.removeVerifiable(streamEntry.proofId);
+                    await this._immutableProofComponent.removeNotarization(streamEntry.proofId);
                     delete streamEntry.proofId;
+                    await MetricHelper.metricIncrement(this._telemetryComponent, AuditableItemStreamMetricIds.ProofsRemovedEntry);
                     // If we are only removing the proof, we need to set the entry
                     // otherwise the entry is going to be removed later anyway.
                     if (removeOnlyProof) {