@enbox/dwn-sdk-js 0.2.2 → 0.3.1

package/src/handlers/messages-sync.ts

@@ -4,6 +4,7 @@ import type { HandlerDependencies, MethodHandler } from '../types/method-handler
 import type { MessagesSyncDiffEntry, MessagesSyncMessage, MessagesSyncReply } from '../types/messages-types.js';
 
 import { authenticate } from '../core/auth.js';
+import { DwnConstant } from '../core/dwn-constant.js';
 import { Encoder } from '../utils/encoder.js';
 import { hashToHex } from '../smt/smt-utils.js';
 import { messageReplyFromError } from '../core/message-reply.js';
@@ -13,11 +14,13 @@ import { PermissionsProtocol } from '../protocols/permissions.js';
 import { DwnError, DwnErrorCode } from '../core/dwn-error.js';
 
 /**
- * Default maximum inline data size for diff responses (256 KB).
+ * Maximum inline data size for diff responses — aligned with the
+ * {@link DwnConstant.maxDataSizeAllowedToBeEncoded} threshold (30 KB).
  * RecordsWrite data payloads smaller than this are base64url-encoded and
- * included directly in the diff reply, avoiding a separate MessagesRead round-trip.
+ * included directly in the diff reply.  Larger payloads must be fetched
+ * separately via MessagesRead.
  */
-const DEFAULT_MAX_INLINE_DATA_SIZE = 262_144;
+const DEFAULT_MAX_INLINE_DATA_SIZE = DwnConstant.maxDataSizeAllowedToBeEncoded;
 
 
 export class MessagesSyncHandler implements MethodHandler {

package/src/handlers/protocols-configure.ts

@@ -83,7 +83,7 @@ export class ProtocolsConfigureHandler implements MethodHandler {
 
       // only emit if the event log is set
       if (this.deps.eventLog !== undefined) {
-        await this.deps.eventLog.emit(tenant, { message }, indexes);
+        await this.deps.eventLog.emit(tenant, { message }, indexes, messageCid);
       }
 
       messageReply = {

package/src/handlers/records-subscribe.ts

@@ -1,9 +1,9 @@
 import type { CoreProtocolRegistry } from '../core/core-protocol.js';
 import type { MessageSort } from '../types/message-types.js';
 import type { MessageStore } from '../types//message-store.js';
-import type { SubscriptionListener } from '../types/subscriptions.js';
 import type { Filter, PaginationCursor } from '../types/query-types.js';
 import type { HandlerDependencies, MethodHandler } from '../types/method-handler.js';
+import type { ProgressGapInfo, SubscriptionListener } from '../types/subscriptions.js';
 import type { RecordsQueryReplyEntry, RecordsSubscribeMessage, RecordsSubscribeReply } from '../types/records-types.js';
 
 import { authenticate } from '../core/auth.js';
@@ -93,6 +93,13 @@ export class RecordsSubscribeHandler implements MethodHandler {
           subscription,
         };
       } catch (error) {
+        if (error instanceof DwnError && error.code === DwnErrorCode.EventLogProgressGap) {
+          const gapInfo = (error as any).gapInfo as ProgressGapInfo | undefined;
+          return {
+            status : { code: 410, detail: 'Progress token gap' },
+            error  : gapInfo !== undefined ? { code: 'ProgressGap' as const, ...gapInfo } : undefined,
+          };
+        }
         return messageReplyFromError(error, 500);
       }
     }

package/src/handlers/records-write.ts

@@ -91,9 +91,32 @@ export class RecordsWriteHandler implements MethodHandler {
     }
 
     if (!incomingMessageIsNewest) {
-      return {
-        status: { code: 409, detail: 'Conflict' }
-      };
+      // Allow re-processing when the existing record was stored as an
+      // initial write without data (isLatestBaseState = false, status 204)
+      // and the incoming message now supplies data.  This happens during
+      // sync when a live pull initially stores the message without data
+      // and a subsequent poll or retry delivers the same message with data.
+      //
+      // We detect the incomplete state by checking whether the existing
+      // message is an initial write that lacks both inline encodedData and
+      // DataStore data — indicating it was stored without data.
+      let existingLacksData = false;
+      if (newestExistingMessage !== undefined && dataStream !== undefined) {
+        const isInitial = await RecordsWrite.isInitialWrite(newestExistingMessage);
+        if (isInitial) {
+          const hasInlineData = !!(newestExistingMessage as any).encodedData;
+          const hasStoredData = this.deps.dataStore
+            ? !!(await this.deps.dataStore.get(tenant, recordsWrite.message.recordId, message.descriptor.dataCid!))
+            : false;
+          existingLacksData = !hasInlineData && !hasStoredData;
+        }
+      }
+
+      if (!existingLacksData) {
+        return {
+          status: { code: 409, detail: 'Conflict' }
+        };
+      }
     }
 
     // Look up the core protocol (if any) for the incoming message so that lifecycle hooks
@@ -142,13 +165,19 @@ export class RecordsWriteHandler implements MethodHandler {
 
       const indexes = await recordsWrite.constructIndexes(isLatestBaseState);
       await this.deps.messageStore.put(tenant, messageWithOptionalEncodedData, indexes);
-      await this.deps.stateIndex!.insert(tenant, await Message.getCid(message), indexes);
+      const messageCid = await Message.getCid(message);
+      await this.deps.stateIndex!.insert(tenant, messageCid, indexes);
 
       // NOTE: We only emit a `RecordsWrite` when the message is the latest base state.
       // Because we allow a `RecordsWrite` which is not the latest state to be written, but not queried, we shouldn't emit it either.
       // It will be emitted as a part of a subsequent next write, if it is the latest base state.
+      //
+      // We emit `messageWithOptionalEncodedData` (not the raw `message`) so
+      // that WebSocket subscribers receive inline `encodedData` for small
+      // records (<= 30 KB).  This allows live sync to store the record
+      // immediately without a separate MessagesRead round-trip.
       if (this.deps.eventLog !== undefined && isLatestBaseState) {
-        await this.deps.eventLog.emit(tenant, { message, initialWrite }, indexes);
+        await this.deps.eventLog.emit(tenant, { message: messageWithOptionalEncodedData, initialWrite }, indexes, messageCid);
       }
     } catch (error) {
       if (error instanceof DwnError) {

package/src/index.ts

@@ -1,6 +1,6 @@
 // export everything that we want to be consumable
 export type { DwnConfig } from './dwn.js';
-export type { EventListener, EventLog, EventLogEntry, EventLogReadOptions, EventLogReadResult, EventLogSubscribeOptions, EventSubscription, MessageEvent, SubscriptionEose, SubscriptionEvent, SubscriptionListener, SubscriptionMessage, SubscriptionReply } from './types/subscriptions.js';
+export type { EventListener, EventLog, EventLogEntry, EventLogReadOptions, EventLogReadResult, EventLogSubscribeOptions, EventSubscription, MessageEvent, ProgressGapInfo, ProgressGapReason, ProgressToken, SubscriptionEose, SubscriptionEvent, SubscriptionListener, SubscriptionMessage, SubscriptionReply } from './types/subscriptions.js';
 export type { AuthorizationModel, Descriptor, DelegatedGrantRecordsWriteMessage, GenericMessage, GenericMessageReply, GenericSignaturePayload, MessageSort, MessageSubscription, Pagination, QueryResultEntry, Status } from './types/message-types.js';
 export type { MessagesFilter, MessagesReadMessage as MessagesReadMessage, MessagesReadReply as MessagesReadReply, MessagesReadReplyEntry as MessagesReadReplyEntry, MessagesReadDescriptor, MessagesSubscribeDescriptor, MessagesSubscribeMessage, MessagesSubscribeReply, MessagesSubscribeMessageOptions, MessagesSyncAction, MessagesSyncDescriptor, MessagesSyncDiffEntry, MessagesSyncMessage, MessagesSyncReply } from './types/messages-types.js';
 export type { GT, LT, Filter, FilterValue, KeyValues, EqualFilter, OneOfFilter, RangeFilter, RangeCriterion, PaginationCursor, QueryOptions, RangeValue, StartsWithFilter } from './types/query-types.js';

package/src/interfaces/messages-subscribe.ts

@@ -1,5 +1,6 @@
 import type { MessagesFilter } from '../types/messages-types.js';
 import type { MessageSigner } from '../types/signer.js';
+import type { ProgressToken } from '../types/subscriptions.js';
 import type { MessagesSubscribeDescriptor, MessagesSubscribeMessage } from '../types/messages-types.js';
 
 import { AbstractMessage } from '../core/abstract-message.js';
@@ -16,10 +17,10 @@ export type MessagesSubscribeOptions = {
   filters?: MessagesFilter[];
   permissionGrantId?: string;
   /**
-   * Opaque EventLog cursor string to resume from. When provided, catch-up events are
-   * replayed from the EventLog and an EOSE marker is delivered before live events.
+   * Progress token to resume from. When provided, catch-up events are replayed
+   * from the EventLog and an EOSE marker is delivered before live events.
    */
-  cursor?: string;
+  cursor?: ProgressToken;
 };
 
 export class MessagesSubscribe extends AbstractMessage<MessagesSubscribeMessage> {

package/src/interfaces/records-subscribe.ts

@@ -1,6 +1,7 @@
 import type { MessageSigner } from '../types/signer.js';
 import type { MessageStore } from '../types/message-store.js';
 import type { Pagination } from '../types/message-types.js';
+import type { ProgressToken } from '../types/subscriptions.js';
 import type { DataEncodedRecordsWriteMessage, DateSort, RecordsFilter, RecordsSubscribeDescriptor, RecordsSubscribeMessage } from '../types/records-types.js';
 
 import { AbstractMessage } from '../core/abstract-message.js';
@@ -23,10 +24,10 @@ export type RecordsSubscribeOptions = {
   protocolRole?: string;
 
   /**
-   * Opaque EventLog cursor string to resume from. When provided, catch-up events are
-   * replayed from the EventLog and an EOSE marker is delivered before live events.
+   * Progress token to resume from. When provided, catch-up events are replayed
+   * from the EventLog and an EOSE marker is delivered before live events.
    */
-  cursor?: string;
+  cursor?: ProgressToken;
 
   /**
    * The delegated grant to sign on behalf of the logical author, which is the grantor (`grantedBy`) of the delegated grant.

package/src/store/storage-controller.ts

@@ -75,7 +75,7 @@ export class StorageController {
 
     // only emit if the event log is set
     if (this.eventLog !== undefined) {
-      await this.eventLog.emit(tenant, { message, initialWrite }, indexes);
+      await this.eventLog.emit(tenant, { message, initialWrite }, indexes, messageCid);
     }
 
     if (message.descriptor.prune) {

package/src/types/messages-types.ts

@@ -1,7 +1,7 @@
 import type { RangeCriterion } from './query-types.js';
-import type { SubscriptionListener } from './subscriptions.js';
 import type { AuthorizationModel, GenericMessage, GenericMessageReply, MessageSubscription } from './message-types.js';
 import type { DwnInterfaceName, DwnMethodName } from '../enums/dwn-interface-method.js';
+import type { ProgressGapInfo, ProgressToken, SubscriptionListener } from './subscriptions.js';
 
 /**
  * filters used when filtering for any type of Message across interfaces
@@ -10,6 +10,12 @@ export type MessagesFilter = {
   interface?: string;
   method?: string;
   protocol?: string;
+  /** Prefix filter for protocolPath. Matches records whose protocolPath equals
+   *  the prefix or starts with the prefix followed by '/'. */
+  protocolPathPrefix?: string;
+  /** Prefix filter for contextId. Matches records whose contextId equals
+   *  the prefix or starts with the prefix followed by '/'. */
+  contextIdPrefix?: string;
   messageTimestamp?: RangeCriterion;
 };
 
@@ -100,6 +106,8 @@ export type MessagesSubscribeMessage = {
 
 export type MessagesSubscribeReply = GenericMessageReply & {
   subscription?: MessageSubscription;
+  /** Present when status.code is 410 — structured gap metadata. */
+  error?: { code: 'ProgressGap' } & ProgressGapInfo;
 };
 
 export type MessagesSubscribeDescriptor = {
@@ -109,9 +117,9 @@ export type MessagesSubscribeDescriptor = {
   filters: MessagesFilter[];
   permissionGrantId?: string;
   /**
-   * Opaque EventLog cursor string to resume from. When provided, the handler replays
-   * events from the EventLog starting after this cursor instead of returning no
+   * Progress token to resume from. When provided, the handler replays events
+   * from the EventLog starting after this position instead of returning no
    * initial snapshot. An EOSE marker is sent after catch-up.
    */
-  cursor?: string;
+  cursor?: ProgressToken;
 };

package/src/types/records-types.ts

@@ -1,9 +1,9 @@
 import type { GeneralJws } from './jws-types.js';
 import type { JweEncryption } from '../utils/encryption.js';
-import type { SubscriptionListener } from './subscriptions.js';
 import type { AuthorizationModel, GenericMessage, GenericMessageReply, GenericSignaturePayload, MessageSubscription, Pagination } from './message-types.js';
 import type { DwnInterfaceName, DwnMethodName } from '../enums/dwn-interface-method.js';
 import type { PaginationCursor, RangeCriterion, RangeFilter, StartsWithFilter } from './query-types.js';
+import type { ProgressGapInfo, ProgressToken, SubscriptionListener } from './subscriptions.js';
 
 export enum DateSort {
   CreatedAscending = 'createdAscending',
@@ -132,11 +132,11 @@ export type RecordsSubscribeDescriptor = {
   dateSort?: DateSort;
   pagination?: Pagination;
   /**
-   * Opaque EventLog cursor string to resume from. When provided, the handler replays
-   * events from the EventLog starting after this cursor instead of querying the
+   * Progress token to resume from. When provided, the handler replays events
+   * from the EventLog starting after this position instead of querying the
    * MessageStore for an initial snapshot. An EOSE marker is sent after catch-up.
    */
-  cursor?: string;
+  cursor?: ProgressToken;
 };
 
 export type RecordsFilter = {
@@ -203,6 +203,8 @@ export type RecordsSubscribeReply = GenericMessageReply & {
   subscription?: MessageSubscription;
   entries?: RecordsQueryReplyEntry[];
   cursor?: PaginationCursor;
+  /** Present when status.code is 410 — structured gap metadata. */
+  error?: { code: 'ProgressGap' } & ProgressGapInfo;
 };
 
 export type RecordsReadMessage = {

package/src/types/subscriptions.ts

@@ -2,6 +2,54 @@ import type { RecordsWriteMessage } from './records-types.js';
 import type { Filter, KeyValues } from './query-types.js';
 import type { GenericMessage, GenericMessageReply, MessageSubscription } from './message-types.js';
 
+// ---------------------------------------------------------------------------
+// ProgressToken — structured cursor for EventLog replay/resume
+// ---------------------------------------------------------------------------
+
+/**
+ * Structured cursor for EventLog replay and resume. Replaces the previous
+ * opaque `string` cursor to provide explicit ordering semantics required
+ * by causal frontier progression in multi-master sync.
+ *
+ * Comparisons are valid only when `streamId` and `epoch` are equal.
+ * `position` is compared numerically (BigInt-safe), never lexicographically.
+ * Progress tokens are source-local: they MUST NOT be reused across different
+ * remote providers or EventLog instances.
+ */
+export type ProgressToken = {
+  /** Stable identity of the source event stream domain. */
+  streamId : string;
+  /** Stream generation/version; changes on non-compatible reset. */
+  epoch : string;
+  /** Monotonic decimal string within `(streamId, epoch)`. Compared numerically. */
+  position : string;
+  /** The CID of the message associated with this event. */
+  messageCid : string;
+};
+
+/**
+ * Reason code for a {@link ProgressGapInfo} — explains why the cursor
+ * cannot be resumed.
+ */
+export type ProgressGapReason = 'token_too_old' | 'epoch_mismatch' | 'stream_mismatch';
+
+/**
+ * Metadata attached to a `DwnError(DwnErrorCode.EventLogProgressGap, ...)`
+ * when an EventLog implementation cannot resume from a given cursor.
+ *
+ * Subscribe handlers translate this into a 410 response.
+ */
+export type ProgressGapInfo = {
+  /** The cursor the consumer requested. */
+  requested : ProgressToken;
+  /** The oldest token still available for replay. */
+  oldestAvailable : ProgressToken;
+  /** The latest token available. */
+  latestAvailable : ProgressToken;
+  /** Why the cursor is no longer valid. */
+  reason : ProgressGapReason;
+};
+
 /**
  * Internal listener type used by {@link EventLog.emit} to notify in-process
  * subscribers. Not intended for direct consumer use — consumers should use
@@ -32,17 +80,16 @@ export type SubscriptionReply = GenericMessageReply & {
 // ---------------------------------------------------------------------------
 
 /**
- * A regular subscription event carrying a message and its EventLog cursor.
+ * A regular subscription event carrying a message and its EventLog progress token.
  */
 export type SubscriptionEvent = {
   type : 'event';
   /**
-   * Opaque cursor string assigned by the EventLog implementation. Clients should
-   * persist this value and pass it back to `subscribe()` or `read()` to resume
-   * from this point. The format is implementation-defined (e.g. numeric sequence
-   * for in-memory, Redis stream ID, NATS stream sequence, etc.).
+   * Structured progress token assigned by the EventLog implementation. Clients
+   * should persist this value and pass it back to `subscribe()` or `read()` to
+   * resume from this point.
    */
-  cursor : string;
+  cursor : ProgressToken;
   /** The event payload (message + optional initialWrite). */
   event : MessageEvent;
 };
@@ -57,10 +104,10 @@ export type SubscriptionEvent = {
 export type SubscriptionEose = {
   type : 'eose';
   /**
-   * Opaque cursor string of the last stored event that was replayed.
+   * Progress token of the last stored event that was replayed.
    * Echoes the input cursor when no stored events matched (i.e. already caught up).
    */
-  cursor : string;
+  cursor : ProgressToken;
 };
 
 /**
@@ -80,15 +127,15 @@ export type SubscriptionListener = (message: SubscriptionMessage) => void;
  */
 export type EventLogSubscribeOptions = {
   /**
-   * Opaque cursor string to resume from (exclusive — events after this cursor
+   * Progress token to resume from (exclusive — events after this position
    * are replayed). When provided, stored events are replayed first, followed by
    * an EOSE marker, then live events. When omitted, only live events are delivered.
    *
-   * Cursor values are implementation-defined and must be obtained from a prior
-   * interaction with the same EventLog instance (e.g. `SubscriptionEvent.cursor`,
-   * `EventLogReadResult.cursor`, or the return value of `emit()`).
+   * Tokens must be obtained from a prior interaction with the same EventLog
+   * instance (e.g. `SubscriptionEvent.cursor`, `EventLogReadResult.cursor`,
+   * or the return value of `emit()`).
    */
-  cursor? : string;
+  cursor? : ProgressToken;
 
   /**
    * Filters evaluated against event indexes. Events must match at least one
@@ -119,8 +166,8 @@ export type EventLogEntry = {
  * Options accepted by {@link EventLog.read}.
  */
 export type EventLogReadOptions = {
-  /** Opaque cursor string to resume from (exclusive — returns events after this cursor). */
-  cursor? : string;
+  /** Progress token to resume from (exclusive — returns events after this position). */
+  cursor? : ProgressToken;
 
   /** Maximum number of events to return. */
   limit? : number;
@@ -137,14 +184,14 @@ export type EventLogReadResult = {
   events : EventLogEntry[];
 
   /**
-   * Opaque cursor string for resuming subsequent reads or subscriptions.
+   * Progress token for resuming subsequent reads or subscriptions.
    *
-   * - When events are returned: cursor of the last event.
+   * - When events are returned: token of the last event.
    * - When no events are returned but a cursor was provided: the input cursor
    *   (meaning "you are caught up, nothing new since this point").
    * - When no events exist and no cursor was provided: `undefined`.
    */
-  cursor? : string;
+  cursor? : ProgressToken;
 };
 
 /**
@@ -162,9 +209,13 @@ export type EventLogReadResult = {
 export interface EventLog {
   /**
    * Persist an event and notify in-process subscribers.
-   * @returns The opaque cursor string assigned to the event, or empty string on failure.
+   * @param tenant     The tenant DID.
+   * @param event      The event payload.
+   * @param indexes    Index values for the event.
+   * @param messageCid The CID of the message being emitted — embedded in the returned token.
+   * @returns A {@link ProgressToken} assigned to the event, or `undefined` on failure.
    */
-  emit(tenant: string, event: MessageEvent, indexes: KeyValues): Promise<string>;
+  emit(tenant: string, event: MessageEvent, indexes: KeyValues, messageCid: string): Promise<ProgressToken | undefined>;
 
   /**
    * Read events from the log starting after `cursor`, optionally filtered.
@@ -188,6 +239,13 @@ export interface EventLog {
    */
   subscribe(tenant: string, id: string, listener: SubscriptionListener, options?: EventLogSubscribeOptions): Promise<EventSubscription>;
 
+  /**
+   * Returns the oldest and latest available progress tokens for a tenant,
+   * or `undefined` if the tenant has no events. Used to construct
+   * `ProgressGap` metadata when a consumer's cursor is no longer replayable.
+   */
+  getReplayBounds(tenant: string): Promise<{ oldest: ProgressToken; latest: ProgressToken } | undefined>;
+
   /**
    * Delete events older than the given sequence number or ISO-8601 timestamp.
    */

package/src/utils/messages.ts

@@ -67,6 +67,29 @@ export class Messages {
       }
 
       messagesQueryFilters.push(this.convertFilter(filter));
+
+      // When protocolPathPrefix is used with a protocol, inject a shadow filter
+      // for ProtocolsConfigure events. Without this, protocol metadata updates
+      // would be excluded (ProtocolsConfigure indexes have no protocolPath).
+      // This mirrors the existing core-protocol additional-filter pattern above.
+      // The messageTimestamp constraint is carried over so time-bounded queries
+      // (including cursor-based subscriptions) also apply to the shadow filter.
+      if ((filter.protocolPathPrefix !== undefined || filter.contextIdPrefix !== undefined) && filter.protocol !== undefined) {
+        const metadataFilter: Filter = {
+          interface : 'Protocols',
+          method    : 'Configure',
+          protocol  : filter.protocol,
+        };
+
+        if (filter.messageTimestamp !== undefined) {
+          const timestampFilter = FilterUtility.convertRangeCriterion(filter.messageTimestamp);
+          if (timestampFilter) {
+            metadataFilter.messageTimestamp = timestampFilter;
+          }
+        }
+
+        messagesQueryFilters.push(metadataFilter);
+      }
     }
 
     return messagesQueryFilters;
@@ -78,12 +101,35 @@ export class Messages {
   private static convertFilter(filter: MessagesFilter): Filter {
     const filterCopy = { ...filter } as Filter;
 
-    const { messageTimestamp } = filter;
+    const { messageTimestamp, protocolPathPrefix, contextIdPrefix } = filter;
     const messageTimestampFilter = messageTimestamp ? FilterUtility.convertRangeCriterion(messageTimestamp) : undefined;
     if (messageTimestampFilter) {
       filterCopy.messageTimestamp = messageTimestampFilter;
       delete filterCopy.dateUpdated;
     }
+
+    // Convert protocolPathPrefix into a protocolPath range filter.
+    // The range gte: prefix, lt: prefix + '/\uffff' matches:
+    //   - exact: 'post' (prefix itself)
+    //   - children: 'post/attachment', 'post/comment', etc.
+    //   - NOT siblings: 'poster', 'postfix' (excluded because '/' < any alphanumeric)
+    if (protocolPathPrefix !== undefined) {
+      delete (filterCopy as any).protocolPathPrefix;
+      filterCopy.protocolPath = {
+        gte : protocolPathPrefix,
+        lt  : protocolPathPrefix + '/\uffff',
+      };
+    }
+
+    // Convert contextIdPrefix into a contextId range filter (same pattern).
+    if (contextIdPrefix !== undefined) {
+      delete (filterCopy as any).contextIdPrefix;
+      filterCopy.contextId = {
+        gte : contextIdPrefix,
+        lt  : contextIdPrefix + '/\uffff',
+      };
+    }
+
     return filterCopy as Filter;
   }
 }