@ibgib/core-gib 0.1.22 → 0.1.25

package/src/sync/sync-saga-message/sync-saga-message-types.mts

@@ -1,8 +1,11 @@
 import { IbGib_V1, IbGibData_V1, IbGibRel8ns_V1 } from "@ibgib/ts-gib/dist/V1/types.mjs";
+import { IbGibAddr } from "@ibgib/ts-gib/dist/types.mjs";
+
 import { KeystoneIbGib_V1 } from "../../keystone/keystone-types.mjs";
 import { SyncStage } from "../sync-constants.mjs";
-import { SyncMode, SyncConflictStrategy, SyncInitData, SyncRequestData, SyncDeltaData, SyncCommitData } from "../sync-types.mjs";
+import { SyncMode, SyncConflictStrategy, } from "../sync-types.mjs";
 import { SYNC_SAGA_MSG_ATOM } from "./sync-saga-message-constants.mjs";
+import type { getDeltaDependencyGraph } from '../../common/other/graph-helper.mjs';
 
 export interface SyncSagaMessageIb_V1 {
     atom: typeof SYNC_SAGA_MSG_ATOM;
@@ -33,41 +36,154 @@ export interface SyncSagaMessageIbGib_V1 extends IbGib_V1<SyncSagaMessageData_V1
 // CONCRETE MESSAGE PAYLOADS (EXTENDING DATA)
 // ===========================================================================
 
-export interface SyncSagaMessageInitData_V1 extends SyncSagaMessageData_V1, SyncInitData {
+/**
+ * The "Hello" of the sync protocol.
+ * 
+ * Exchanges knowledge vectors (what I have) and identity (who I am).
+ */
+export interface SyncSagaMessageInitData_V1 extends SyncSagaMessageData_V1 {
     stage: typeof SyncStage.init;
     stones: string[];
+    /**
+     * Map of TJP (Temporal Joint Point) to latest known ibGibAddr.
+     * Use this to calculate what is missing (Delta).
+     * 
+     * IOW, this is a map of the starting and end points of an ibGib's timeline.
+     */
+    knowledgeVector: { [tjp: string]: IbGibAddr };
+
+    /**
+     * The Keystone Identity of the sender.
+     * Required for the first handshake or if the receiver doesn't know the sender.
+     */
+    identity?: KeystoneIbGib_V1;
+
+    /**
+     * What the sender wants to do.
+     * e.g. "push" (I have data for you) or "pull" (Give me data).
+     * @default 'push'
+     */
+    mode?: SyncMode;
+
+    /**
+     * This is also in the sync saga ibgib itself now...
+     * 
+     * @see {@link SyncOptions.conflictStrategy}
+     */
+    conflictStrategy?: SyncConflictStrategy;
+}
+
+/**
+ * When an endpoing doesn't have a timeline or some part of it, then this info 
+ * contains sufficient information for the other endpoint to know what addrs to
+ * send.
+ */
+export interface SyncSagaRequestAddrInfo {
+    /**
+     * We know this addr exists, but we don't have it. If we don't know anything
+     * about this timeline, then this will equal {@link tjpAddr}
+     */
+    addr: IbGibAddr;
+    /**
+     * if we're talking about an ibgib with a timeline (with ibgib.rel8ns.tjp),
+     * then this will be that timeline's addr (tjpAddr).
+     */
+    tjpAddr?: IbGibAddr;
+    /**
+     * If this is truthy, then we already have something from this timeline and
+     * this will be the latest addr in that timeline.
+     * 
+     * If falsy, then we are requesting the entire timeline be sent.
+     * 
+     * This is to be used to calculate the delta graph.
+     * @see {@link getDeltaDependencyGraph}
+     */
+    latestAddrAlreadyHave?: IbGibAddr;
+}
+
+/**
+ * when one endpoint says "hey, i have these that you need", they create this.
+ */
+export interface SyncSagaPushOfferInfo {
+    /**
+     * all addrs to be pushed, should be the entire delta dependency graph.
+     * @see {@link getDeltaDependencyGraph}
+     */
+    addrs: IbGibAddr[];
+    /**
+     * if we're talking about an ibgib with a timeline (with ibgib.rel8ns.tjp),
+     * then this will be that timeline's addr (tjpAddr).
+     */
+    tjpAddr?: IbGibAddr;
+}
+
+export interface SyncSagaConflictInfo {
+    tjpAddr: string;
+    /**
+     * "local" is relative to the POV of the creator of the conflict info
+     */
+    localAddr: string;
+    /**
+     * "remote" is relative to the POV of the creator of the conflict info
+     */
+    remoteAddr: string;
+    /**
+     * Full history of the timeline from the Receiver's perspective.
+     * Used by Sender to find LCA.
+     */
+    timelineAddrs: string[];
+    /**
+     * string description
+     */
+    reason: string;
+    /**
+     * if true, this conflict should abort the sync
+     */
+    terminal: boolean;
 }
 
 export interface SyncSagaMessageAckData_V1 extends SyncSagaMessageData_V1 {
     stage: typeof SyncStage.ack;
-    deltaReqAddrs: string[];
-    pushOfferAddrs: string[];
+    deltaRequestAddrInfos: SyncSagaRequestAddrInfo[];
+    pushOfferInfos: SyncSagaPushOfferInfo[];
     /**
-     * Map of group keys (TJP or Constant Addr) to list of known addresses (Full History).
-     * Used by Sender to calculate differential payloads.
+     * Map of group keys (TJP addr/constant addr) to list of known addresses
+     * in that timeline.
+     * 
+     * This will be used by Sender to calculate differential payloads. So 
+     * when the sender goes to get the dependency graph of its tip ibgib,
+     * it can skip all of the addrs in this.
+     * 
+     * For example, say sender has A^5 and receiver only has A^3. Sender 
+     * will see in this map { [A^0]: [A^0, A^1, A^2, A^3] } and know that
+     * it needs all of the dependencies of A^5 (including A^4), but it does
+     * NOT need to send A^0-A^3 (and dependencies) because receiver already has
+     * these.
+     * 
+     * ## notes
+     * 
+     * I dislike this name in my rewrite/cleanup of this algorithm. However, I'm
+     * keeping it and am using it right now in handleInitFrame for including
+     * the receiver's latest addr/tip for the deltaReqAddrs. But really, I should 
      */
-    knowledgeVector?: { [groupKey: string]: string[] };
+    // knowledgeVector?: { [tjpAddr: string]: string[] };
     /**
      * List of identified conflicts.
      * 
-     * If present, the Sender should use the `timelineAddrs` (Receiver's history) 
-     * to compute the LCA and delta requirements for merging.
-     */
-    conflicts?: {
-        tjpAddr: string;
-        localAddr: string;
-        remoteAddr: string;
-        /**
-         * Full history of the timeline from the Receiver's perspective.
-         * Used by Sender to find LCA.
-         */
-        timelineAddrs: string[];
-        reason: string;
-        terminal: boolean;
-    }[];
+     * If present, the Sender should use the `timelineAddrs` (Receiver's
+     * history) to compute the LCA and delta requirements for merging.
+     */
+    conflicts?: SyncSagaConflictInfo[];
 }
 
-export interface SyncSagaMessageDeltaData_V1 extends SyncSagaMessageData_V1, SyncDeltaData {
+/**
+ * The "Body" of the sync protocol. Carries the manifests of data to be
+ * transferred. 
+ * 
+ * If any {@link payloadAddrsDomain} exist, then these will be populated in the
+ * {@link SyncSagaContextIbGib_V1}.
+ */
+export interface SyncSagaMessageDeltaData_V1 extends SyncSagaMessageData_V1 {
     stage: typeof SyncStage.delta;
     requests?: string[];
     /**
@@ -75,9 +191,29 @@ export interface SyncSagaMessageDeltaData_V1 extends SyncSagaMessageData_V1, Syn
      * and is ready to commit the transaction.
      */
     proposeCommit?: boolean;
+    /**
+     * Domain ibgib addresses (actual user data being synced).
+     * These go to tempSpace ONLY until final commit.
+     */
+    payloadAddrsDomain?: string[];
 }
 
-export interface SyncSagaMessageCommitData_V1 extends SyncSagaMessageData_V1, SyncCommitData {
+/**
+ * The "Seal" of the sync protocol.
+ */
+export interface SyncSagaMessageCommitData_V1 extends SyncSagaMessageData_V1 {
     stage: typeof SyncStage.commit;
+    /**
+     * True if the session completed successfully from the sender's perspective.
+     */
+    success: boolean;
+    /**
+     * Optional verification metadata (e.g. success count).
+     */
+    successCount?: number;
+    /**
+     * If failed, the reasons why.
+     */
+    errors?: string[];
 }
 

package/src/sync/sync-types.mts

@@ -1,10 +1,15 @@
+import { IbGibAddr } from "@ibgib/ts-gib/dist/types.mjs";
 import { IbGib_V1, IbGibData_V1, IbGibRel8ns_V1 } from "@ibgib/ts-gib/dist/V1/types.mjs";
 
-import { KeystoneIbGib_V1, KeystoneProof } from "../keystone/keystone-types.mjs";
-import { SYNC_ATOM, SyncStage } from "./sync-constants.mjs";
+import { SubjectWitness } from "../common/pubsub/subject/subject-types.mjs";
+import { SyncSagaContextIbGib_V1 } from "./sync-saga-context/sync-saga-context-types.mjs";
+import { KeystoneIbGib_V1, } from "../keystone/keystone-types.mjs";
+import { SYNC_ATOM, SYNC_MSG_REL8N_NAME, } from "./sync-constants.mjs";
 import { IbGibSpaceAny } from "../witness/space/space-base-v1.mjs";
 import { MetaspaceService } from "../witness/space/metaspace/metaspace-types.mjs";
 import { SyncPeerWitness } from "./sync-peer/sync-peer-types.mjs";
+import { FlatIbGibGraph } from "../common/other/graph-types.mjs";
+import { SyncSagaConflictInfo } from "./sync-saga-message/sync-saga-message-types.mjs";
 
 
 // #region SyncMode
@@ -29,15 +34,19 @@ export function isValidSyncMode(mode: string): mode is SyncMode {
 // #region SyncConflictStrategy
 export const SYNC_CONFLICT_STRATEGY_ABORT = 'abort';
 export const SYNC_CONFLICT_STRATEGY_OPTIMISTIC = 'optimistic';
-export const SYNC_CONFLICT_STRATEGY_MANUAL = 'manual';
+// export const SYNC_CONFLICT_STRATEGY_MANUAL = 'manual'; // not implemented yet
 export type SyncConflictStrategy =
     | typeof SYNC_CONFLICT_STRATEGY_ABORT
     | typeof SYNC_CONFLICT_STRATEGY_OPTIMISTIC
-    | typeof SYNC_CONFLICT_STRATEGY_MANUAL;
+    // | typeof SYNC_CONFLICT_STRATEGY_MANUAL // not implemented yet
+    ;
+/**
+ * @see {@link SyncOptions.conflictStrategy}
+ */
 export const SyncConflictStrategy = {
     abort: SYNC_CONFLICT_STRATEGY_ABORT,
     optimistic: SYNC_CONFLICT_STRATEGY_OPTIMISTIC,
-    manual: SYNC_CONFLICT_STRATEGY_MANUAL,
+    // manual: SYNC_CONFLICT_STRATEGY_MANUAL, // not implemented yet
 } satisfies { [key in SyncConflictStrategy]: SyncConflictStrategy };
 export const SYNC_CONFLICT_STRATEGY_VALID_VALUES = Object.values(SyncConflictStrategy);
 export function isValidSyncConflictStrategy(strategy: string): strategy is SyncConflictStrategy {
@@ -45,6 +54,48 @@ export function isValidSyncConflictStrategy(strategy: string): strategy is SyncC
 }
 // #endregion SyncConflictStrategy
 
+export interface NextSagaFrameInfo_Frame {
+    frame: SyncIbGib_V1;
+    payloadIbGibsDomain?: IbGib_V1[];
+    conflictInfos?: SyncSagaConflictInfo;
+    responseWasNull: undefined;
+}
+export interface NextSagaFrameInfo_Null {
+    frame: never;
+    payloadIbGibsDomain?: never;
+    conflictInfos?: never;
+    responseWasNull: true;
+}
+/**
+ * Result of handling a saga frame.
+ *
+ * ## notes
+ *
+ * After commit, this may have been a null response, so `frame` will not be
+ * defined, thus the discriminated union type.
+ */
+export type NextSagaFrameInfo =
+    NextSagaFrameInfo_Frame | NextSagaFrameInfo_Null;
+
+
+export interface HandleSagaResponseContextResult_Base {
+    /**
+     * if truthy, handling saga context failed, else it succeeded.
+     */
+    errorMsg?: string;
+    /**
+     * inner handling of the frame itself
+     */
+    nextFrameInfo?: NextSagaFrameInfo;
+}
+export interface HandleSagaResponseContextResult_Frame extends HandleSagaResponseContextResult_Base {
+    nextFrameInfo: NextSagaFrameInfo;
+}
+export interface HandleSagaResponseContextResult_Error extends HandleSagaResponseContextResult_Base {
+    errorMsg: string;
+}
+export type HandleSagaResponseContextResult = HandleSagaResponseContextResult_Frame | HandleSagaResponseContextResult_Error;
+
 export interface SyncOptions {
     /**
      * The peer we are syncing with.
@@ -52,7 +103,7 @@ export interface SyncOptions {
     peer: SyncPeerWitness;
     /**
      * The ibgibs we wish to sync.
-     * 
+     *
      * These should all exist in {@link source}
      */
     domainIbGibs: IbGib_V1[],
@@ -81,110 +132,71 @@ export interface SyncOptions {
      */
     identitySecret?: string;
     /**
-     * How to handle conflicts when both Source and Dest have diverged on the same timeline.
+     * How to handle conflicts when both Source and Dest have diverged on the
+     * same timeline.
+     *
      * @default 'abort'
      */
     conflictStrategy?: SyncConflictStrategy;
     /**
-     * @deprecated Use `identity` instead if you have a specific identity.
-     * If true, creates an ephemeral session identity.
+     * If true, creates an ephemeral session identity for the sync process to
+     * secure the sync transaction itself.
+     *
      * @default true
      */
     useSessionIdentity?: boolean;
 }
 
-// ===========================================================================
-// PROTOCOL PAYLOADS
-// ===========================================================================
-
-/**
- * The "Hello" of the sync protocol.
- * Exchanges knowledge vectors (what I have) and identity (who I am).
- */
-export interface SyncInitData {
+export interface SyncSagaInfo {
     /**
-     * Map of TJP (Temporal Joint Point) to latest known ibGibAddr.
-     * Use this to calculate what is missing (Delta).
-     * 
-     * IOW, this is a map of the starting and end points of an ibGib's timeline.
+     * The unique ID of the saga.
      */
-    knowledgeVector: { [tjp: string]: string };
+    sagaId: string;
 
     /**
-     * The Keystone Identity of the sender.
-     * Required for the first handshake or if the receiver doesn't know the sender.
+     * Observable stream of context updates happening during the saga.
+     *
+     * Subscribe to this to receive real-time progress throughout the sync
+     * process, whenever the sync ibgib is evolved (including when receiving a
+     * response from the peer).
      */
-    identity?: KeystoneIbGib_V1;
+    updates$: SubjectWitness<SyncSagaContextIbGib_V1>;
 
     /**
-     * What the sender wants to do.
-     * e.g. "push" (I have data for you) or "pull" (Give me data).
-     * @default 'push'
+     * Promise that resolves when the saga completes successfully.
      */
-    mode?: SyncMode;
-    conflictStrategy?: SyncConflictStrategy;
+    done: Promise<void>;
 }
 
-/**
- * A specific request for data.
- * Used for "pull" based deltas or fixing gaps (packet loss).
- */
-export interface SyncRequestData {
-    /**
-     * The ibGib addresses requested.
-     */
-    addrs: string[];
+export interface SyncSagaFrameDependencyGraph {
+    sagaIbGib: SyncIbGib_V1;
+    msgStones: IbGib_V1[];
+    identities: KeystoneIbGib_V1[];
 }
 
 /**
- * The "Body" of the sync protocol.
- * Carries the manifests of data to be transferred.
+ * given some group of domain ibgibs, this is info extracted from them.
  */
-export interface SyncDeltaData {
+export interface DomainIbGibAnalysisInfo {
     /**
-     * Control ibgib addresses (sync frames, stones, keystones).
-     * These go to BOTH destSpace + tempSpace for audit trail.
+     * flat array of stones (ibgibs without dna)
      */
-    payloadAddrsControl?: string[];
-
+    stones: IbGib_V1[];
     /**
-     * Domain ibgib addresses (actual user data being synced).
-     * These go to tempSpace ONLY until final commit.
+     * map of tjp -> sorted timeline ibgib
      */
-    payloadAddrsDomain?: string[];
-
+    timelinesMap: { [tjp: string]: IbGib_V1[] };
     /**
-     * THIS SHOULD NOT EXIST. NOT BE DEPRECATED. JUST REMOVED!!
-     * 
-     * @deprecated Use payloadAddrsControl + payloadAddrsDomain instead.
-     * Legacy combined payload addresses.
+     * sorted by dependency/interdependency
      */
-    payloadAddrs?: string[];
-}
-
-/**
- * The "Seal" of the sync protocol.
- * The authority is derived from the `identity` relation on the Keystone Frame.
- */
-export interface SyncCommitData {
+    topologicallySortedTjpAddrs: IbGibAddr[];
     /**
-     * True if the session completed successfully from the sender's perspective.
-     */
-    success: boolean;
-    /**
-     * Optional verification metadata (e.g. success count).
-     */
-    successCount?: number;
-    /**
-     * If failed, the reasons why.
+     * full dependency graph of domain ibgibs
      */
-    errors?: string[];
+    fullGraph: FlatIbGibGraph;
 }
 
-// ===========================================================================
-// IBGIB DATA STRUCTURES
-// ===========================================================================
-
+// #region Sync Ib, Data, Rel8ns, IbGib
 export interface SyncIb_V1 {
     atom: typeof SYNC_ATOM;
     uuid: string;
@@ -218,11 +230,6 @@ export interface SyncData_V1 extends IbGibData_V1 {
      * 2. Rate limiting / Backpressure (Don't process this old request).
      */
     expirationTimestamp?: string;
-
-    /**
-     * Polymorphic payload based on stage.
-     */
-    payload?: SyncInitData | SyncRequestData | SyncDeltaData | SyncCommitData;
 }
 
 export interface SyncRel8ns_V1 extends IbGibRel8ns_V1 {
@@ -231,30 +238,14 @@ export interface SyncRel8ns_V1 extends IbGibRel8ns_V1 {
      * This MUST point to the specific Keystone Frame that authorizes this sync frame.
      */
     identity?: string[];
-}
-
-export interface SyncIbGib_V1 extends IbGib_V1<SyncData_V1, SyncRel8ns_V1> { }
-
-/**
- * Information returned when starting a Sync Saga.
- * Allows the caller to subscribe to updates and await completion.
- */
-import { SubjectWitness } from "../common/pubsub/subject/subject-types.mjs";
-import { SyncSagaContextIbGib_V1 } from "./sync-saga-context/sync-saga-context-types.mjs";
-
-export interface SyncSagaInfo {
-    /**
-     * The unique ID of the saga.
-     */
-    sagaId: string;
-    /**
-     * Stream of context updates happening during the saga.
-     * Subscribe to this to receive real-time progress.
-     */
-    updates$: SubjectWitness<SyncSagaContextIbGib_V1>;
 
     /**
-     * Promise that resolves when the saga completes successfully.
+     * The message stone that contains the information about the particular
+     * stage of the sync process we are in.
      */
-    done: Promise<void>;
+    [SYNC_MSG_REL8N_NAME]: IbGibAddr[];
 }
+
+export interface SyncIbGib_V1 extends IbGib_V1<SyncData_V1, SyncRel8ns_V1> { }
+
+// #endregion Sync Ib, Data, Rel8ns, IbGib

package/src/timeline/timeline-api.mts

@@ -27,6 +27,8 @@ import { SpecialIbGibType } from "../common/other/other-types.mjs";
 import { getSpecialConfigKey, toDto } from "../common/other/ibgib-helper.mjs";
 import { execInSpaceWithLocking, getFromSpace, persistTransformResult } from "../witness/space/space-helper.mjs";
 import { TimelineHistoryInfo } from "./timeline-types.mjs";
+import { toFlatGraph } from "../common/other/graph-helper.mjs";
+import { FlatIbGibGraph } from "../common/other/graph-types.mjs";
 
 const logalot = GLOBAL_LOG_A_LOT;
 
@@ -845,7 +847,16 @@ export async function createTimeline<TData extends IbGibData_V1 = IbGibData_V1>(
             dna: noDna ? false : true,
             nCounter: true,
             squash: noDna ? true : false,
-            tjp: { timestamp: true, uuid: true },
+            tjp: {
+                /**
+                 * always use the timestamp to generate a unique timeline
+                 */
+                timestamp: true,
+                /**
+                 * only generate new uuid if it's not in the incoming data
+                 */
+                uuid: !data?.uuid,
+            },
         });
 
         // Persist the new timeline and its DNA to the designated space.
@@ -938,21 +949,17 @@ export async function getHistory({
         space ??= await metaspace.getLocalUserSpace({ lock: false });
         if (!space) { throw new Error(`(UNEXPECTED) space is required. (E: e3d1c0b1a9a8b7c6d5e4f3a2b1a0b9c8)`); }
 
-        const ibGibsToMap = (ibgibs: IbGib_V1[]) => {
-            const map: { [addr: IbGibAddr]: IbGib_V1 } = {};
-            ibgibs.forEach(x => { map[getIbGibAddr({ ibGib: x })] = x; });
-            return map;
-        }
-
         const headIbGib = getLatest ?
             await getLatestTimelineIbGibDto_nonLocking({ timeline, metaspace, space }) :
             toDto({ ibGib: timeline }) as IbGib_V1;
         const headAddr = getIbGibAddr({ ibGib: headIbGib });
 
-        let pastIbGibMap: { [addr: IbGibAddr]: IbGib_V1 } = {};
+        let pastIbGibMap: FlatIbGibGraph = {};
 
         const dnaMap = includeDna ?
-            ibGibsToMap(await getDna({ ibGib: headIbGib, metaspace, space })) :
+            toFlatGraph({
+                ibGibs: await getDna({ ibGib: headIbGib, metaspace, space })
+            }) :
             undefined;
 
         const pastAddrs = timeline.rel8ns?.past ?? [];
@@ -971,7 +978,7 @@ export async function getHistory({
             throw new Error(`Could not get all dna ibGibs for ${headAddr}. Addrs not found: ${notFound.join(', ')}. Errors: ${resGet.errorMsg ?? '[unknown error (E: 887339d8b5580ed1ac9892a554255525)]'} (E: 4ab7a8a58adff4e238171f7c110c6825)`);
         }
 
-        pastIbGibMap = ibGibsToMap(resGet.ibGibs!);
+        pastIbGibMap = toFlatGraph({ ibGibs: resGet.ibGibs! }) ?? {};
 
         const orderedPastIbGibs: IbGib_V1[] = [];
         for (const pastAddr of pastAddrs) {