@enbox/agent 0.5.10 → 0.5.12

package/src/sync-messages.ts

@@ -1,5 +1,6 @@
 import type { EnboxPlatformAgent } from './types/agent.js';
 import type { PermissionsApi } from './types/permissions.js';
+import type { PushResult } from './types/sync.js';
 import type { GenericMessage, MessagesReadReply, MessagesSyncDiffEntry, UnionMessageReply } from '@enbox/dwn-sdk-js';
 
 import { DwnInterfaceName, DwnMethodName, Encoder, Message } from '@enbox/dwn-sdk-js';
@@ -36,6 +37,20 @@ export function syncMessageReplyIsSuccessful(reply: UnionMessageReply): boolean
     );
 }
 
+/**
+ * Determines whether a failed push reply represents a permanent failure that
+ * should NOT be retried. Permanent failures include protocol violations (400),
+ * authorization errors (401/403), and schema validation errors that will never
+ * succeed regardless of retry.
+ *
+ * Transient failures (5xx, network errors) are worth retrying.
+ */
+export function isPermanentPushFailure(reply: UnionMessageReply): boolean {
+  return reply.status.code === 400 ||
+    reply.status.code === 401 ||
+    reply.status.code === 403;
+}
+
 /**
  * Helper to get the CID of a message for logging purposes.
  */
@@ -296,6 +311,10 @@ export async function fetchRemoteMessages({ did, dwnUrl, delegateDid, protocol,
  * Messages are fetched first, then sorted in dependency order (topological sort)
  * so that initial writes come before updates, and ProtocolsConfigures come before
  * records that reference those protocols.
+ *
+ * Returns a {@link PushResult} with per-CID outcome tracking instead of throwing
+ * on the first failure. Callers use this to advance the push checkpoint
+ * incrementally — only up to the highest contiguous success.
  */
 export async function pushMessages({ did, dwnUrl, delegateDid, protocol, messageCids, agent, permissionsApi }: {
   did: string;
@@ -305,13 +324,20 @@ export async function pushMessages({ did, dwnUrl, delegateDid, protocol, message
   messageCids: string[];
   agent: EnboxPlatformAgent;
   permissionsApi: PermissionsApi;
-}): Promise<void> {
+}): Promise<PushResult> {
+  const succeeded: string[] = [];
+  const failed: string[] = [];
+  const permanentlyFailed: string[] = [];
+
   // Step 1: Fetch all local messages (streams are pull-based, not yet consumed).
   const fetched: SyncMessageEntry[] = [];
   for (const messageCid of messageCids) {
     const dwnMessage = await getLocalMessage({ author: did, messageCid, delegateDid, protocol, agent, permissionsApi });
     if (dwnMessage) {
       fetched.push(dwnMessage);
+    } else {
+      // Message could not be fetched locally — mark as failed.
+      failed.push(messageCid);
     }
   }
 
@@ -320,6 +346,7 @@ export async function pushMessages({ did, dwnUrl, delegateDid, protocol, message
 
   // Step 3: Push messages in dependency order, consuming each stream as we go.
   for (const entry of sorted) {
+    const cid = await getMessageCid(entry.message);
     try {
       const reply = await agent.rpc.sendDwnRequest({
         dwnUrl,
@@ -328,16 +355,27 @@ export async function pushMessages({ did, dwnUrl, delegateDid, protocol, message
         message   : entry.message
       });
 
-      if (!syncMessageReplyIsSuccessful(reply)) {
-        const cid = await getMessageCid(entry.message);
+      if (syncMessageReplyIsSuccessful(reply)) {
+        succeeded.push(cid);
+      } else if (isPermanentPushFailure(reply)) {
+        // Permanent failures (400/401/403) will never succeed — do NOT retry.
+        // These include protocol violations (RecordLimitExceeded), auth errors,
+        // and schema validation failures.
+        console.warn(`SyncEngineLevel: push permanently failed for ${cid}: ${reply.status.code} ${reply.status.detail}`);
+        permanentlyFailed.push(cid);
+      } else {
+        // Transient failures (5xx, etc.) — worth retrying.
         console.error(`SyncEngineLevel: push failed for ${cid}: ${reply.status.code} ${reply.status.detail}`);
+        failed.push(cid);
       }
     } catch (error: any) {
-      // Preserve the original error so callers can diagnose the root cause.
-      const detail = error.message ?? error;
-      throw new Error(`SyncEngineLevel: push to ${dwnUrl} failed: ${detail}`);
+      // Network errors — transient, worth retrying.
+      console.error(`SyncEngineLevel: push error for ${cid}: ${error.message ?? error}`);
+      failed.push(cid);
     }
   }
+
+  return { succeeded, failed, permanentlyFailed };
 }
 
 /**

package/src/sync-replication-ledger.ts

@@ -0,0 +1,197 @@
+import type { AbstractLevel } from 'abstract-level';
+import type { ProgressToken } from '@enbox/dwn-sdk-js';
+
+import type { DirectionCheckpoint, LinkStatus, ReplicationLinkState, SyncScope } from './types/sync.js';
+
+import { computeScopeId } from './types/sync.js';
+
+/** Separator used in compound LevelDB keys. */
+const KEY_SEP = '^';
+
+/**
+ * Durable replication ledger — persists {@link ReplicationLinkState} for each
+ * sync link in a LevelDB sublevel. Provides CRUD operations and replication
+ * checkpoint helpers.
+ *
+ * Key format: `{tenantDid}^{remoteEndpoint}^{scopeId}`
+ *
+ * Each link tracks independent pull and push {@link DirectionCheckpoint}s.
+ * The ledger does not own subscriptions or timers — it is a passive state
+ * store called by the sync engine.
+ */
+export class ReplicationLedger {
+  private readonly db: AbstractLevel<string | Buffer | Uint8Array>;
+  private sublevel;
+
+  constructor(db: AbstractLevel<string | Buffer | Uint8Array>) {
+    this.db = db;
+    this.sublevel = this.db.sublevel('replicationLinks');
+  }
+
+  // ---------------------------------------------------------------------------
+  // Key helpers
+  // ---------------------------------------------------------------------------
+
+  /** Build the compound key for a link. */
+  private static buildKey(tenantDid: string, remoteEndpoint: string, scopeId: string): string {
+    return `${tenantDid}${KEY_SEP}${remoteEndpoint}${KEY_SEP}${scopeId}`;
+  }
+
+  // Note: compound keys use raw '^' separator. This is safe because tenantDid
+  // (DID URI), remoteEndpoint (URL), and scopeId (base64url hash) cannot
+  // contain '^'. If future fields can contain '^', keys must be escaped.
+
+  // ---------------------------------------------------------------------------
+  // CRUD
+  // ---------------------------------------------------------------------------
+
+  /**
+   * Get-or-create a link. If the link does not exist, it is created with
+   * `initializing` status and empty checkpoints.
+   */
+  public async getOrCreateLink(params: {
+    tenantDid : string;
+    remoteEndpoint : string;
+    scope : SyncScope;
+    delegateDid? : string;
+    protocol? : string;
+  }): Promise<ReplicationLinkState> {
+    const scopeId = await computeScopeId(params.scope);
+    const key = ReplicationLedger.buildKey(params.tenantDid, params.remoteEndpoint, scopeId);
+
+    try {
+      const raw = await this.sublevel.get(key);
+      const link = JSON.parse(raw) as ReplicationLinkState;
+      // connectivity is runtime state — always reset to 'unknown' on load
+      // so stale 'online' from a previous session doesn't give false positives.
+      link.connectivity = 'unknown';
+      return link;
+    } catch (error) {
+      const e = error as { code: string };
+      if (e.code !== 'LEVEL_NOT_FOUND') {
+        throw error;
+      }
+    }
+
+    // Create a new link with empty checkpoints.
+    const link: ReplicationLinkState = {
+      tenantDid      : params.tenantDid,
+      remoteEndpoint : params.remoteEndpoint,
+      scopeId,
+      scope          : params.scope,
+      status         : 'initializing',
+      connectivity   : 'unknown',
+      pull           : {},
+      push           : {},
+      delegateDid    : params.delegateDid,
+      protocol       : params.protocol,
+    };
+
+    await this.sublevel.put(key, JSON.stringify(link));
+    return link;
+  }
+
+  /** Persist the current state of a link. */
+  public async saveLink(link: ReplicationLinkState): Promise<void> {
+    const key = ReplicationLedger.buildKey(link.tenantDid, link.remoteEndpoint, link.scopeId);
+    link.lastActivityAt = new Date().toISOString();
+    await this.sublevel.put(key, JSON.stringify(link));
+  }
+
+  /** Delete a link. */
+  public async deleteLink(tenantDid: string, remoteEndpoint: string, scopeId: string): Promise<void> {
+    const key = ReplicationLedger.buildKey(tenantDid, remoteEndpoint, scopeId);
+    await this.sublevel.del(key);
+  }
+
+  /** List all links for a tenant. */
+  public async getLinksForTenant(tenantDid: string): Promise<ReplicationLinkState[]> {
+    const prefix = `${tenantDid}${KEY_SEP}`;
+    const links: ReplicationLinkState[] = [];
+    for await (const [key, value] of this.sublevel.iterator()) {
+      if (key.startsWith(prefix)) {
+        links.push(JSON.parse(value) as ReplicationLinkState);
+      }
+    }
+    return links;
+  }
+
+  /** List all links. */
+  public async getAllLinks(): Promise<ReplicationLinkState[]> {
+    const links: ReplicationLinkState[] = [];
+    for await (const [, value] of this.sublevel.iterator()) {
+      links.push(JSON.parse(value) as ReplicationLinkState);
+    }
+    return links;
+  }
+
+  // ---------------------------------------------------------------------------
+  // Status transitions
+  // ---------------------------------------------------------------------------
+
+  /** Transition a link to a new status and persist. */
+  public async setStatus(link: ReplicationLinkState, status: LinkStatus): Promise<void> {
+    link.status = status;
+    await this.saveLink(link);
+  }
+
+  // ---------------------------------------------------------------------------
+  // Minimal checkpoint helpers (durable state only — no progression logic)
+  // ---------------------------------------------------------------------------
+
+  /**
+   * Compare two tokens by position (BigInt numeric comparison).
+   * Returns negative if a < b, zero if equal, positive if a > b.
+   * Caller must verify streamId and epoch match before calling.
+   */
+  public static comparePosition(a: ProgressToken, b: ProgressToken): number {
+    const diff = BigInt(a.position) - BigInt(b.position);
+    if (diff < BigInt(0)) { return -1; }
+    if (diff > BigInt(0)) { return 1; }
+    return 0;
+  }
+
+  /**
+   * Check whether a token belongs to the same domain (streamId + epoch) as
+   * the checkpoint's current baseline. Returns `true` if domains match or if
+   * the checkpoint has no baseline yet.
+   */
+  public static validateTokenDomain(checkpoint: DirectionCheckpoint, token: ProgressToken): boolean {
+    if (checkpoint.contiguousAppliedToken === undefined) { return true; }
+    return token.streamId === checkpoint.contiguousAppliedToken.streamId &&
+           token.epoch === checkpoint.contiguousAppliedToken.epoch;
+  }
+
+  /**
+   * Update `receivedToken` to the highest seen token (for observability).
+   * Does NOT advance `contiguousAppliedToken` — that is owned by the engine's
+   * delivery-order tracking.
+   */
+  public static setReceivedToken(checkpoint: DirectionCheckpoint, token: ProgressToken): void {
+    if (
+      checkpoint.receivedToken === undefined ||
+      ReplicationLedger.comparePosition(token, checkpoint.receivedToken) > 0
+    ) {
+      checkpoint.receivedToken = token;
+    }
+  }
+
+  /**
+   * Commit a token as the new contiguous applied baseline. The caller (engine)
+   * must have already verified that all earlier delivered tokens for this link
+   * are durably committed before calling this.
+   */
+  public static commitContiguousToken(checkpoint: DirectionCheckpoint, token: ProgressToken): void {
+    checkpoint.contiguousAppliedToken = token;
+  }
+
+  /**
+   * Reset a replication checkpoint (e.g., after repair or domain change).
+   * Clears all state.
+   */
+  public static resetCheckpoint(checkpoint: DirectionCheckpoint, token?: ProgressToken): void {
+    checkpoint.contiguousAppliedToken = token;
+    checkpoint.receivedToken = token;
+  }
+}
+

package/src/types/sync.ts

@@ -1,3 +1,5 @@
+import type { ProgressToken } from '@enbox/dwn-sdk-js';
+
 import type { EnboxPlatformAgent } from './agent.js';
 
 /**
@@ -25,6 +27,179 @@ export type SyncConnectivityState = 'online' | 'offline' | 'unknown';
  */
 export type SyncMode = 'poll' | 'live';
 
+// ---------------------------------------------------------------------------
+// Sync scope and scope identity
+// ---------------------------------------------------------------------------
+
+/**
+ * Describes what a replication link syncs. Currently whole-tenant only
+ * (`kind: 'full'`). Scoped subset sync (`kind: 'protocol'` with
+ * `protocolPathPrefixes` / `contextIdPrefixes`) is deferred to Phase 3.
+ */
+export type SyncScope = {
+  /** Scope kind. Only `'full'` is implemented in Phase 1. */
+  kind: 'full';
+} | {
+  /**
+   * Protocol-scoped sync. Deferred to Phase 3 — included here for type
+   * forward-compatibility only.
+   */
+  kind: 'protocol';
+  protocol: string;
+  protocolPathPrefixes?: string[];
+  contextIdPrefixes?: string[];
+};
+
+/**
+ * Computes a deterministic, collision-resistant identifier for a {@link SyncScope}.
+ *
+ * The ID is `base64url(SHA-256(canonicalJSON))` where `canonicalJSON` is the
+ * scope object with keys sorted alphabetically and array values sorted
+ * lexicographically.
+ *
+ * Used as part of the LevelDB key for the replication ledger:
+ * `{tenantDid}^{remoteEndpoint}^{scopeId}`.
+ */
+export async function computeScopeId(scope: SyncScope): Promise<string> {
+  const canonical: Record<string, unknown> = { kind: scope.kind };
+  if (scope.kind === 'protocol') {
+    canonical.protocol = scope.protocol;
+    if (scope.protocolPathPrefixes !== undefined) {
+      canonical.protocolPathPrefixes = [...new Set(scope.protocolPathPrefixes)].sort();
+    }
+    if (scope.contextIdPrefixes !== undefined) {
+      canonical.contextIdPrefixes = [...new Set(scope.contextIdPrefixes)].sort();
+    }
+  }
+
+  // Stable JSON: keys sorted by construction order (kind < protocol < protocolPathPrefixes).
+  const json = JSON.stringify(canonical);
+  const bytes = new TextEncoder().encode(json);
+  const hashBuffer = await crypto.subtle.digest('SHA-256', bytes);
+  const hashArray = new Uint8Array(hashBuffer);
+
+  // base64url encode (no padding).
+  let base64 = '';
+  for (const b of hashArray) {
+    base64 += String.fromCharCode(b);
+  }
+  return btoa(base64).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
+}
+
+// ---------------------------------------------------------------------------
+// Replication checkpoint types
+// ---------------------------------------------------------------------------
+
+/**
+ * Maximum number of in-flight deliveries (runtime ordinals) a link may
+ * accumulate before transitioning to `repairing`. This is the overflow
+ * threshold for the engine's in-memory delivery tracker, not for durable
+ * checkpoint state. Normative per the sync redesign RFC.
+ */
+export const MAX_PENDING_TOKENS = 100;
+
+/**
+ * Tracks directional (pull or push) replay progression for a single
+ * replication link. All tokens belong to the same `(streamId, epoch)`.
+ *
+ * This is the **durable** replication checkpoint persisted to the ledger.
+ * In-memory delivery-order tracking (ordinals, in-flight commits) is owned
+ * by the sync engine and is not persisted — on crash recovery, replay
+ * restarts from `contiguousAppliedToken` and idempotent apply handles
+ * any re-delivered events.
+ */
+export type DirectionCheckpoint = {
+  /**
+   * The latest token received from the source (pull) or confirmed by the
+   * remote (push). May be ahead of `contiguousAppliedToken` when events
+   * arrive out of order. Used for observability.
+   */
+  receivedToken?: ProgressToken;
+
+  /**
+   * The highest token such that all earlier delivered tokens for this link
+   * have been durably applied. This is the resume point after crash/reconnect.
+   *
+   * Advancement is controlled by the engine's delivery-order tracking,
+   * not by position arithmetic. Positions may be sparse (filtered streams).
+   */
+  contiguousAppliedToken?: ProgressToken;
+};
+
+/**
+ * Status of a replication link.
+ *
+ * - `initializing` — link created, no subscriptions open yet.
+ * - `live` — actively receiving events via subscription.
+ * - `repairing` — gap detected or pending overflow; running SMT reconciliation.
+ * - `degraded_poll` — subscription failed; polling at reduced frequency.
+ * - `paused` — explicitly paused by the application.
+ */
+export type LinkStatus = 'initializing' | 'live' | 'repairing' | 'degraded_poll' | 'paused';
+
+/**
+ * Durable state of a single replication link. Persisted to LevelDB and
+ * loaded on startup. Each link is identified by the tuple
+ * `(tenantDid, remoteEndpoint, scopeId)`.
+ */
+export type ReplicationLinkState = {
+  /** The tenant DID this link syncs for. */
+  tenantDid: string;
+
+  /** The remote DWN endpoint URL. */
+  remoteEndpoint: string;
+
+  /** Deterministic hash of the {@link SyncScope}. See {@link computeScopeId}. */
+  scopeId: string;
+
+  /** The scope definition this link covers. */
+  scope: SyncScope;
+
+  /** Current link status. */
+  status: LinkStatus;
+
+  /** Pull-direction replication checkpoint (remote → local). */
+  pull: DirectionCheckpoint;
+
+  /** Push-direction replication checkpoint (local → remote). */
+  push: DirectionCheckpoint;
+
+  /** Per-link connectivity state. Used to compute the aggregate engine-level state. */
+  connectivity: SyncConnectivityState;
+
+  /** Delegate DID used to sign sync messages, if any. */
+  delegateDid?: string;
+
+  /**
+   * Protocol filter for this link, if any. Duplicates the protocol in `scope`
+   * for operational convenience — used by permission lookups and cursor key
+   * building. The scope is the source of truth for what to sync; this field
+   * is the source of truth for how to authenticate. To be consolidated in
+   * Phase 3 when scope resolution is more complex.
+   */
+  protocol?: string;
+
+  /** ISO-8601 timestamp of last successful sync activity. */
+  lastActivityAt?: string;
+};
+
+// ---------------------------------------------------------------------------
+// Push result (per-CID outcome tracking)
+// ---------------------------------------------------------------------------
+
+/**
+ * Result of a batch push operation. Replaces the previous throw-on-first-failure
+ * pattern so callers can advance the push replication checkpoint incrementally.
+ */
+export type PushResult = {
+  /** messageCids that were accepted (202/204/409 — idempotent success). */
+  succeeded: string[];
+  /** messageCids that failed with a transient error (5xx, network) — worth retrying. */
+  failed: string[];
+  /** messageCids that failed permanently (400/401/403) — will never succeed, do NOT retry. */
+  permanentlyFailed: string[];
+};
+
 /**
  * Parameters for {@link SyncEngine.startSync}.
  */
@@ -54,6 +229,28 @@ export type StartSyncParams = {
   interval?: string;
 };
 
+// ---------------------------------------------------------------------------
+// Sync observability events
+// ---------------------------------------------------------------------------
+
+/**
+ * Events emitted by the sync engine at key state transitions.
+ * Consumers subscribe via `SyncEngine.on('event', handler)` and can
+ * hook these into metrics, logging, or UI state.
+ */
+export type SyncEvent =
+  | { type: 'link:status-change'; tenantDid: string; remoteEndpoint: string; protocol?: string; from: LinkStatus; to: LinkStatus }
+  | { type: 'link:connectivity-change'; tenantDid: string; remoteEndpoint: string; protocol?: string; from: SyncConnectivityState; to: SyncConnectivityState }
+  | { type: 'checkpoint:pull-advance'; tenantDid: string; remoteEndpoint: string; protocol?: string; position: string; messageCid: string }
+  | { type: 'checkpoint:push-advance'; tenantDid: string; remoteEndpoint: string; protocol?: string; position: string; messageCid: string }
+  | { type: 'repair:started'; tenantDid: string; remoteEndpoint: string; protocol?: string; attempt: number }
+  | { type: 'repair:completed'; tenantDid: string; remoteEndpoint: string; protocol?: string }
+  | { type: 'repair:failed'; tenantDid: string; remoteEndpoint: string; protocol?: string; attempt: number; error: string }
+  | { type: 'degraded-poll:entered'; tenantDid: string; remoteEndpoint: string; protocol?: string }
+  | { type: 'gap:detected'; tenantDid: string; remoteEndpoint: string; protocol?: string; reason: string };
+
+export type SyncEventListener = (event: SyncEvent) => void;
+
 export interface SyncEngine {
   /**
    * The agent that the SyncEngine is attached to.
@@ -107,6 +304,13 @@ export interface SyncEngine {
    */
   stopSync(timeout?: number): Promise<void>;
 
+  /**
+   * Subscribe to sync engine events. Returns an unsubscribe function.
+   * Events are emitted at key state transitions: checkpoint advancement,
+   * link status changes, repair, degraded_poll, gap detection.
+   */
+  on(listener: SyncEventListener): () => void;
+
   /**
    * Release all resources held by the sync engine (LevelDB handles, timers,
    * WebSocket subscriptions). After calling `close()`, the engine should not