@starkeep/sync-engine 0.1.0

package/dist/index.d.ts

@@ -0,0 +1,373 @@
+import * as _starkeep_protocol_primitives from '@starkeep/protocol-primitives';
+import { HLCTimestamp, StarkeepId, AnyRecord, HLCClock, StarkeepError } from '@starkeep/protocol-primitives';
+import { ObjectStorageAdapter, DatabaseAdapter } from '@starkeep/storage-adapter';
+import { DatabaseSync } from 'node:sqlite';
+import { IncomingMessage, ServerResponse } from 'node:http';
+
+interface AppSyncableTableInfo {
+    readonly name: string;
+    readonly pkColumns: string[];
+}
+interface AppSyncableNamespace {
+    readonly appId: string;
+    readonly tables: AppSyncableTableInfo[];
+    readonly filesEnabled: boolean;
+    /** Derived from tables — convenience accessor. */
+    readonly tableNames: string[];
+}
+interface AppSyncableNamespaceStore {
+    get(appId: string): AppSyncableNamespace | null;
+    list(): AppSyncableNamespace[];
+}
+interface AppSyncableRowEntry {
+    readonly timestamp: HLCTimestamp;
+    readonly appId: string;
+    /** Bare table name (no engine prefix on the wire). */
+    readonly table: string;
+    readonly op: "insert" | "update" | "delete";
+    readonly row?: Record<string, unknown>;
+    readonly where?: Record<string, unknown>;
+}
+interface AppSyncableApplier {
+    apply(entry: AppSyncableRowEntry): Promise<void> | void;
+}
+/** Pagination options for `ScanCapableApplier.scanSince`. */
+interface ScanSinceOptions {
+    /** Max rows to return in this page. */
+    readonly limit?: number;
+    /**
+     * Serialized HLC of the last row returned by the previous page. The next
+     * page returns rows with `updated_at > cursor`. When omitted, the page
+     * starts from `sinceHlcStr`.
+     */
+    readonly cursor?: string;
+}
+/** Page returned by `ScanCapableApplier.scanSince`. */
+interface ScanSincePage {
+    readonly rows: AppSyncableRowEntry[];
+    /**
+     * Cursor to pass on the next call to continue the scan. `null` when no
+     * further rows exist past this page.
+     */
+    readonly nextCursor: string | null;
+    readonly hasMore: boolean;
+}
+/**
+ * Optional capability that appliers can implement to support exchange
+ * synthesis. Scans rows with `updated_at > sinceHlcStr` (the global floor)
+ * in HLC order, paginated by cursor so the engine can stop after `limit`
+ * matches without buffering the whole table.
+ */
+interface ScanCapableApplier extends AppSyncableApplier {
+    scanSince(appId: string, table: string, sinceHlcStr: string, options?: ScanSinceOptions): Promise<ScanSincePage>;
+}
+/**
+ * A row read from the framework-owned `_starkeep_sync_records` table.
+ * Mirrors the column shape declared in `@starkeep/shared-space-api`'s
+ * `FILE_RECORDS_COLUMNS` plus the always-appended HLC bookkeeping columns.
+ * The sync engine's file-transfer pass derives upload/download decisions
+ * from blob presence (`localObjectStorage.has(key)`), not from any stored
+ * status — there is no `sync_status` column on this row. See
+ * `residency.ts` (`RecordResidency`, `residencyOf`) for the named derived
+ * state, and `system-design.md` "Per-record residency" for the rationale.
+ */
+interface FileRecordRow {
+    readonly id: string;
+    readonly object_storage_key: string;
+    readonly content_hash: string;
+    readonly mime_type: string;
+    readonly size_bytes: number;
+    readonly original_filename: string | null;
+    readonly origin_app_id: string;
+    readonly created_at: string;
+    readonly updated_at: string;
+    readonly deleted_at: string | null;
+}
+type Watermarks = Record<string, HLCTimestamp>;
+interface SyncExchangeRequest {
+    /** Caller's view of what it has seen per nodeId. */
+    readonly watermarks: Watermarks;
+    /** Records the caller believes the peer hasn't seen yet. */
+    readonly records?: AnyRecord[];
+    /** App-syncable row deltas the caller believes the peer hasn't seen. */
+    readonly appSyncableRows?: AppSyncableRowEntry[];
+    /** Max records the responder should ship in this round. */
+    readonly limit?: number;
+}
+interface SyncExchangeResponse {
+    /** Records the caller hasn't seen (`updated_at > callerWatermarks[nodeId]`). */
+    readonly records: AnyRecord[];
+    /** Same delta logic per app schema. */
+    readonly appSyncableRows: AppSyncableRowEntry[];
+    readonly hasMore: boolean;
+}
+interface SyncTransport {
+    exchange(request: SyncExchangeRequest): Promise<SyncExchangeResponse>;
+}
+interface FileSyncManifest {
+    readonly fileHash: string;
+    readonly objectStorageKey: string;
+    readonly sizeBytes: number;
+    readonly mimeType?: string;
+}
+interface FileEntry {
+    readonly key: string;
+    readonly mimeType?: string;
+}
+interface FileSyncEngine {
+    /** True if a transferFile for this key is currently running in this process. */
+    isTransferInFlight(key: string): boolean;
+    getFilesToPush(localStorage: ObjectStorageAdapter, remoteStorage: ObjectStorageAdapter, entries: FileEntry[]): Promise<FileSyncManifest[]>;
+    getFilesToPull(localStorage: ObjectStorageAdapter, remoteStorage: ObjectStorageAdapter, entries: FileEntry[]): Promise<FileSyncManifest[]>;
+    /**
+     * Resolve true on a successful transfer (or if the source key is now present
+     * at the destination). Resolves false if the transfer is already in flight
+     * or the source file doesn't exist.
+     */
+    transferFile(manifest: FileSyncManifest, source: ObjectStorageAdapter, destination: ObjectStorageAdapter): Promise<boolean>;
+}
+type ChangeEventType = "remote-update-available" | "local-data-synced" | "local-change-recorded";
+interface ChangeEvent {
+    readonly eventType: ChangeEventType;
+    readonly recordIds: StarkeepId[];
+    readonly timestamp: HLCTimestamp;
+    /**
+     * For `local-change-recorded`: the appId whose sync channel owns this write.
+     * Set when an app-specific data write happens (the calling app); left unset
+     * for shared-record writes (those are owned by the always-on Drive channel
+     * by Shape-A convention, which is a deployment fact the SDK doesn't name).
+     * The sync supervisor uses this to nudge only the affected engine.
+     */
+    readonly originAppId?: string;
+}
+type ChangeListener = (event: ChangeEvent) => void;
+interface ChangeNotifier {
+    subscribe(listener: ChangeListener): () => void;
+    emit(event: ChangeEvent): void;
+}
+interface SyncStateStore {
+    /** Caller's "what I've seen per nodeId" — advanced by records actually applied from peers. */
+    getWatermarks(): Promise<Watermarks>;
+    setWatermarks(watermarks: Watermarks): Promise<void>;
+    /**
+     * Last-known peer-side watermarks, returned by the peer on the previous
+     * exchange. Used by the caller to compute outbound deltas without an extra
+     * round-trip. Defaults to {} on first exchange.
+     */
+    getPeerWatermarks(): Promise<Watermarks>;
+    setPeerWatermarks(watermarks: Watermarks): Promise<void>;
+    getHlcClockState(): Promise<{
+        wallTime: number;
+        counter: number;
+    } | null>;
+    setHlcClockState(state: {
+        wallTime: number;
+        counter: number;
+    }): Promise<void>;
+}
+interface ExchangeResult {
+    readonly applied: number;
+    readonly shipped: number;
+    readonly hasMore: boolean;
+}
+interface SyncEngine {
+    /**
+     * One version-vector exchange round with the peer:
+     *   1. Read own + last-known peer watermarks
+     *   2. For each outbound record (peer hasn't seen): push its blob if any,
+     *      then ship metadata. Blob push failure excludes that record from the
+     *      round; peerWatermarks stays behind it for an automatic retry.
+     *   3. For each inbound record: apply metadata, then pull its blob if any.
+     *      Blob pull failure leaves own watermark behind it; next round the
+     *      responder still ships it.
+     *   4. Persist updated watermarks.
+     */
+    exchange(): Promise<ExchangeResult>;
+    readonly changeNotifier: ChangeNotifier;
+}
+interface SyncEngineOptions {
+    readonly localDatabaseAdapter: DatabaseAdapter;
+    readonly localObjectStorage: ObjectStorageAdapter;
+    readonly remoteObjectStorage: ObjectStorageAdapter;
+    readonly transport: SyncTransport;
+    readonly clock: _starkeep_protocol_primitives.HLCClock;
+    readonly syncState?: SyncStateStore;
+    /**
+     * Provides the applier (for applying incoming exchange rows) and namespace
+     * store (for scanning local rows on the outbound side). Without it,
+     * app-syncable rows are silently skipped on both directions.
+     */
+    readonly appSyncableSource?: {
+        readonly namespaces: AppSyncableNamespaceStore;
+        readonly applier: AppSyncableApplier & ScanCapableApplier;
+    };
+    /**
+     * Channel split. When true (default), this engine ships and applies
+     * shared records (SR — the `shared.records` table). The always-on Starkeep
+     * Drive channel sets this true and provides no `appSyncableSource`, so it is
+     * the *only* channel that carries shared records. Per-app channels set this
+     * false and carry only their own app-specific (`appSyncableSource`) rows, so
+     * shared-record sync is identical regardless of which apps are cloud-installed.
+     */
+    readonly syncSharedRecords?: boolean;
+    /**
+     * Max items per exchange round, applied to both the outbound local scan and
+     * the inbound request limit. Default 1000. Tests use small values (e.g. 5)
+     * to exercise multi-round pagination without seeding thousands of records;
+     * production callers may tune this against poll frequency / throughput.
+     */
+    readonly pageLimit?: number;
+    /**
+     * Internal page size for the cursor-paginated outbound scan loop. Default
+     * 500. Tests can set this small to force the cursor to advance across
+     * multiple DB queries within one exchange round (otherwise a small test
+     * dataset fits in a single page and the cursor never moves).
+     */
+    readonly scanPageSize?: number;
+}
+
+interface SqliteSyncStateStoreOptions {
+    readonly db: DatabaseSync;
+}
+declare function createSqliteSyncStateStore(options: SqliteSyncStateStoreOptions): SyncStateStore;
+
+declare function createChangeNotifier(): ChangeNotifier;
+
+/** Advance `watermarks[hlc.nodeId]` to `max(current, hlc)`. */
+declare function advanceWatermark(watermarks: Watermarks, hlc: HLCTimestamp): void;
+/** Merge `incoming` into `into`, taking the max per nodeId. */
+declare function mergeWatermarks(into: Watermarks, incoming: Watermarks): Watermarks;
+/** Watermark for `nodeId`, or `ZERO_HLC` if unseen. */
+declare function watermarkFor(watermarks: Watermarks, nodeId: string): HLCTimestamp;
+/**
+ * Return records the peer hasn't seen yet, judged against `peerWatermarks`:
+ * `record.updatedAt > peerWatermarks[record.updatedAt.nodeId] ?? ZERO_HLC`.
+ */
+declare function selectUnseen<T extends {
+    updatedAt: HLCTimestamp;
+}>(records: T[], peerWatermarks: Watermarks): T[];
+
+declare function createFileSyncEngine(): FileSyncEngine;
+
+/**
+ * Sync engine: drives one version-vector exchange round per tick.
+ *
+ * Blob transfer is gated on the same watermark that drives metadata transfer.
+ * A record's blob is pushed before its metadata ships; a record's blob is
+ * pulled before its receipt is acknowledged. If either fails, the watermark
+ * doesn't advance past it, and the next round naturally retries.
+ *
+ * Shared records (SR) and app-record rows in the reserved `_starkeep_sync_records`
+ * table (AR) are interleaved per nodeId in HLC order so the contiguous-prefix
+ * watermark rule covers both streams: a blob failure on an AR row blocks any
+ * later SR record on the same nodeId from shipping in the same round (and vice
+ * versa). Without that, the per-nodeId watermark could leapfrog a failed item.
+ *
+ * There is no scan-everything reconciliation pass. There is no `sync_status`.
+ * Steady state issues zero storage HEAD requests: the watermark delta tells
+ * us exactly which records (and therefore which blobs) need attention.
+ */
+declare function createSyncEngine(options: SyncEngineOptions): SyncEngine;
+
+/**
+ * Per-record state on a single side, derived from facts already on disk.
+ * There is intentionally no persisted `sync_status` column; this type names
+ * what the combination of (row presence, blob presence, deletedAt) means.
+ *
+ * See system-design.md "Per-record residency" for the full rationale and how
+ * the watermark serves as the durable backstop for the Staged state.
+ *
+ * - absent     — no row for this id on this side.
+ * - staged     — row present, blob required, blob not yet present locally.
+ * - resident   — row present, blob present locally.
+ * - tombstoned — `deletedAt` is set. Propagates like resident; blob GC is a
+ *                separate concern.
+ */
+type RecordResidency = "absent" | "staged" | "resident" | "tombstoned";
+/**
+ * Classify a record's residency on this side. Pass `null` for `recordRow` to
+ * model "row not present" (returns `absent`).
+ *
+ * This is the single canonical derivation. Code and tests should call it
+ * rather than reconstructing the predicate from `localStorage.has(key)` etc.
+ *
+ * Note: rows in `_starkeep_sync_records` always have a blob (the table's
+ * purpose). Records that opt out of file storage live in app-syncable
+ * metadata tables instead and don't reach this function.
+ */
+declare function residencyOf(recordRow: FileRecordRow | null, localStorage: ObjectStorageAdapter): Promise<RecordResidency>;
+
+interface InProcessTransportOptions {
+    readonly databaseAdapter: DatabaseAdapter;
+    readonly clock: HLCClock;
+    /**
+     * When provided, the transport synthesizes app-syncable row entries on
+     * exchange (by scanning updated_at per table) and applies incoming rows on
+     * apply (LWW UPSERT).
+     */
+    readonly appSyncableSource?: {
+        readonly namespaces: AppSyncableNamespaceStore;
+        readonly applier: AppSyncableApplier;
+    };
+    /**
+     * Object storage backing the records this transport serves. Used only as a
+     * reference for the file-transfer pass elsewhere; the exchange protocol
+     * itself does no blob inspection.
+     */
+    readonly objectStorage: ObjectStorageAdapter;
+    /**
+     * Channel split (responder side). When true (default), this transport
+     * applies and scans shared records (the `shared.records` table). The
+     * cloud-side Drive channel sets this true with no `appSyncableSource`; per-app
+     * channels set it false and serve only that app's app-specific rows. Mirrors
+     * `SyncEngineOptions.syncSharedRecords` on the requester side.
+     */
+    readonly syncSharedRecords?: boolean;
+}
+/**
+ * `SyncTransport` that talks directly to an in-process database adapter.
+ * Used for tests and for running a "cloud" side in the same Node process.
+ *
+ * Exchange semantics:
+ *   - Apply incoming records via `put(snapshot)` with HLC LWW.
+ *   - Scan local records the caller hasn't seen (per-nodeId watermark filter).
+ *   - Return `responderWatermarks` = MAX(updated_at) per nodeId.
+ *
+ * Conflict resolution is pure HLC LWW — no rejected[], no OCC.
+ */
+declare function createInProcessSyncTransport(options: InProcessTransportOptions): SyncTransport;
+
+interface HttpSyncTransportOptions {
+    readonly baseUrl: string;
+    readonly fetch?: typeof globalThis.fetch;
+    readonly getAuthHeader?: () => string | undefined;
+}
+/**
+ * `SyncTransport` that talks to a remote Starkeep-compatible HTTP server
+ * over `fetch`. Single endpoint: `POST {baseUrl}/sync/exchange`.
+ */
+declare function createHttpSyncTransport(options: HttpSyncTransportOptions): SyncTransport;
+
+interface HttpSyncServerOptions {
+    readonly databaseAdapter: DatabaseAdapter;
+    readonly objectStorageAdapter: ObjectStorageAdapter;
+    readonly clock: HLCClock;
+    /**
+     * Optional transport override — if provided, takes precedence over the
+     * default in-process transport.
+     */
+    readonly transport?: SyncTransport;
+}
+type Handler = (req: IncomingMessage, res: ServerResponse) => Promise<boolean>;
+/**
+ * Request handler that recognizes the Starkeep sync + file routes.
+ * Returns `true` if the request was handled; callers can compose this with
+ * their own routing layer.
+ */
+declare function createHttpSyncHandler(options: HttpSyncServerOptions): Handler;
+
+declare class SyncError extends StarkeepError {
+    constructor(message: string, cause?: unknown);
+}
+
+export { type AppSyncableApplier, type AppSyncableNamespace, type AppSyncableNamespaceStore, type AppSyncableRowEntry, type AppSyncableTableInfo, type ChangeEvent, type ChangeEventType, type ChangeListener, type ChangeNotifier, type ExchangeResult, type FileEntry, type FileRecordRow, type FileSyncEngine, type FileSyncManifest, type HttpSyncServerOptions, type HttpSyncTransportOptions, type RecordResidency, type ScanCapableApplier, type ScanSinceOptions, type ScanSincePage, type SyncEngine, type SyncEngineOptions, SyncError, type SyncExchangeRequest, type SyncExchangeResponse, type SyncStateStore, type SyncTransport, type Watermarks, advanceWatermark, createChangeNotifier, createFileSyncEngine, createHttpSyncHandler, createHttpSyncTransport, createInProcessSyncTransport, createSqliteSyncStateStore, createSyncEngine, mergeWatermarks, residencyOf, selectUnseen, watermarkFor };