@ibgib/core-gib 0.1.21 → 0.1.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/common/other/graph-helper.d.mts +8 -0
- package/dist/common/other/graph-helper.d.mts.map +1 -1
- package/dist/common/other/graph-helper.mjs +31 -1
- package/dist/common/other/graph-helper.mjs.map +1 -1
- package/dist/sync/sync-conflict.respec.mjs +31 -28
- package/dist/sync/sync-conflict.respec.mjs.map +1 -1
- package/dist/sync/sync-constants.d.mts +21 -11
- package/dist/sync/sync-constants.d.mts.map +1 -1
- package/dist/sync/sync-constants.mjs +18 -6
- package/dist/sync/sync-constants.mjs.map +1 -1
- package/dist/sync/sync-helpers.d.mts +24 -15
- package/dist/sync/sync-helpers.d.mts.map +1 -1
- package/dist/sync/sync-helpers.mjs +163 -92
- package/dist/sync/sync-helpers.mjs.map +1 -1
- package/dist/sync/sync-innerspace-constants.respec.mjs +12 -7
- package/dist/sync/sync-innerspace-constants.respec.mjs.map +1 -1
- package/dist/sync/sync-innerspace-deep-updates.respec.mjs +12 -7
- package/dist/sync/sync-innerspace-deep-updates.respec.mjs.map +1 -1
- package/dist/sync/sync-innerspace-dest-ahead.respec.mjs +15 -10
- package/dist/sync/sync-innerspace-dest-ahead.respec.mjs.map +1 -1
- package/dist/sync/sync-innerspace-multiple-timelines.respec.mjs +13 -7
- package/dist/sync/sync-innerspace-multiple-timelines.respec.mjs.map +1 -1
- package/dist/sync/sync-innerspace-partial-update.respec.mjs +15 -7
- package/dist/sync/sync-innerspace-partial-update.respec.mjs.map +1 -1
- package/dist/sync/sync-innerspace.respec.mjs +13 -7
- package/dist/sync/sync-innerspace.respec.mjs.map +1 -1
- package/dist/sync/sync-peer/sync-peer-constants.d.mts +2 -0
- package/dist/sync/sync-peer/sync-peer-constants.d.mts.map +1 -0
- package/dist/sync/sync-peer/sync-peer-constants.mjs +2 -0
- package/dist/sync/sync-peer/sync-peer-constants.mjs.map +1 -0
- package/dist/sync/sync-peer/sync-peer-helpers.d.mts +2 -0
- package/dist/sync/sync-peer/sync-peer-helpers.d.mts.map +1 -0
- package/dist/sync/sync-peer/sync-peer-helpers.mjs +2 -0
- package/dist/sync/sync-peer/sync-peer-helpers.mjs.map +1 -0
- package/dist/sync/sync-peer/sync-peer-innerspace/sync-peer-innerspace-constants.d.mts +8 -0
- package/dist/sync/sync-peer/sync-peer-innerspace/sync-peer-innerspace-constants.d.mts.map +1 -0
- package/dist/sync/sync-peer/sync-peer-innerspace/sync-peer-innerspace-constants.mjs +8 -0
- package/dist/sync/sync-peer/sync-peer-innerspace/sync-peer-innerspace-constants.mjs.map +1 -0
- package/dist/sync/sync-peer/sync-peer-innerspace/sync-peer-innerspace-helpers.d.mts +18 -0
- package/dist/sync/sync-peer/sync-peer-innerspace/sync-peer-innerspace-helpers.d.mts.map +1 -0
- package/dist/sync/sync-peer/sync-peer-innerspace/sync-peer-innerspace-helpers.mjs +54 -0
- package/dist/sync/sync-peer/sync-peer-innerspace/sync-peer-innerspace-helpers.mjs.map +1 -0
- package/dist/sync/sync-peer/sync-peer-innerspace/sync-peer-innerspace-types.d.mts +65 -0
- package/dist/sync/sync-peer/sync-peer-innerspace/sync-peer-innerspace-types.d.mts.map +1 -0
- package/dist/sync/sync-peer/sync-peer-innerspace/sync-peer-innerspace-types.mjs +5 -0
- package/dist/sync/sync-peer/sync-peer-innerspace/sync-peer-innerspace-types.mjs.map +1 -0
- package/dist/sync/sync-peer/sync-peer-innerspace/sync-peer-innerspace-v1.d.mts +44 -0
- package/dist/sync/sync-peer/sync-peer-innerspace/sync-peer-innerspace-v1.d.mts.map +1 -0
- package/dist/sync/sync-peer/sync-peer-innerspace/sync-peer-innerspace-v1.mjs +183 -0
- package/dist/sync/sync-peer/sync-peer-innerspace/sync-peer-innerspace-v1.mjs.map +1 -0
- package/dist/sync/sync-peer/sync-peer-types.d.mts +28 -0
- package/dist/sync/sync-peer/sync-peer-types.d.mts.map +1 -1
- package/dist/sync/sync-peer/sync-peer-v1.d.mts +51 -9
- package/dist/sync/sync-peer/sync-peer-v1.d.mts.map +1 -1
- package/dist/sync/sync-peer/sync-peer-v1.mjs +244 -26
- package/dist/sync/sync-peer/sync-peer-v1.mjs.map +1 -1
- package/dist/sync/sync-saga-context/sync-saga-context-helpers.d.mts +26 -10
- package/dist/sync/sync-saga-context/sync-saga-context-helpers.d.mts.map +1 -1
- package/dist/sync/sync-saga-context/sync-saga-context-helpers.mjs +119 -30
- package/dist/sync/sync-saga-context/sync-saga-context-helpers.mjs.map +1 -1
- package/dist/sync/sync-saga-context/sync-saga-context-types.d.mts +31 -22
- package/dist/sync/sync-saga-context/sync-saga-context-types.d.mts.map +1 -1
- package/dist/sync/sync-saga-context/sync-saga-context-types.mjs +1 -9
- package/dist/sync/sync-saga-context/sync-saga-context-types.mjs.map +1 -1
- package/dist/sync/sync-saga-coordinator.d.mts +81 -87
- package/dist/sync/sync-saga-coordinator.d.mts.map +1 -1
- package/dist/sync/sync-saga-coordinator.mjs +627 -571
- package/dist/sync/sync-saga-coordinator.mjs.map +1 -1
- package/dist/sync/sync-saga-message/sync-saga-message-types.d.mts +105 -22
- package/dist/sync/sync-saga-message/sync-saga-message-types.d.mts.map +1 -1
- package/dist/sync/sync-types.d.mts +56 -76
- package/dist/sync/sync-types.d.mts.map +1 -1
- package/dist/sync/sync-types.mjs +5 -0
- package/dist/sync/sync-types.mjs.map +1 -1
- package/dist/timeline/timeline-api.d.mts.map +1 -1
- package/dist/timeline/timeline-api.mjs +15 -8
- package/dist/timeline/timeline-api.mjs.map +1 -1
- package/dist/witness/light-witness-base-v1.d.mts.map +1 -1
- package/dist/witness/light-witness-base-v1.mjs +2 -0
- package/dist/witness/light-witness-base-v1.mjs.map +1 -1
- package/dist/witness/space/inner-space/inner-space-v1.mjs +1 -1
- package/dist/witness/space/inner-space/inner-space-v1.mjs.map +1 -1
- package/package.json +1 -1
- package/src/common/other/graph-helper.mts +26 -1
- package/src/sync/README.md +31 -22
- package/src/sync/sync-conflict.respec.mts +31 -26
- package/src/sync/sync-constants.mts +19 -9
- package/src/sync/sync-helpers.mts +173 -97
- package/src/sync/sync-innerspace-constants.respec.mts +12 -7
- package/src/sync/sync-innerspace-deep-updates.respec.mts +12 -7
- package/src/sync/sync-innerspace-dest-ahead.respec.mts +14 -9
- package/src/sync/sync-innerspace-multiple-timelines.respec.mts +13 -7
- package/src/sync/sync-innerspace-partial-update.respec.mts +15 -7
- package/src/sync/sync-innerspace.respec.mts +13 -7
- package/src/sync/sync-peer/sync-peer-constants.mts +0 -0
- package/src/sync/sync-peer/sync-peer-helpers.mts +0 -0
- package/src/sync/sync-peer/sync-peer-innerspace/sync-peer-innerspace-constants.mts +8 -0
- package/src/sync/sync-peer/sync-peer-innerspace/sync-peer-innerspace-helpers.mts +72 -0
- package/src/sync/sync-peer/sync-peer-innerspace/sync-peer-innerspace-types.mts +72 -0
- package/src/sync/sync-peer/sync-peer-innerspace/sync-peer-innerspace-v1.mts +193 -0
- package/src/sync/sync-peer/sync-peer-types.mts +30 -1
- package/src/sync/sync-peer/sync-peer-v1.mts +229 -30
- package/src/sync/sync-saga-context/sync-saga-context-helpers.mts +140 -43
- package/src/sync/sync-saga-context/sync-saga-context-types.mts +34 -30
- package/src/sync/sync-saga-coordinator.mts +678 -660
- package/src/sync/sync-saga-message/sync-saga-message-types.mts +106 -22
- package/src/sync/sync-types.mts +59 -87
- package/src/timeline/timeline-api.mts +17 -10
- package/src/witness/light-witness-base-v1.mts +2 -1
- package/src/witness/space/inner-space/inner-space-v1.mts +1 -1
- package/test_output.log +0 -0
- package/tmp.md +62 -44
- package/dist/sync/sync-local-spaces.respec.d.mts +0 -2
- package/dist/sync/sync-local-spaces.respec.d.mts.map +0 -1
- package/dist/sync/sync-local-spaces.respec.mjs +0 -159
- package/dist/sync/sync-local-spaces.respec.mjs.map +0 -1
- package/dist/sync/sync-peer/sync-peer-innerspace-v1.d.mts +0 -42
- package/dist/sync/sync-peer/sync-peer-innerspace-v1.d.mts.map +0 -1
- package/dist/sync/sync-peer/sync-peer-innerspace-v1.mjs +0 -194
- package/dist/sync/sync-peer/sync-peer-innerspace-v1.mjs.map +0 -1
- package/dist/sync/sync-saga-coordinator.respec.d.mts +0 -2
- package/dist/sync/sync-saga-coordinator.respec.d.mts.map +0 -1
- package/dist/sync/sync-saga-coordinator.respec.mjs +0 -40
- package/dist/sync/sync-saga-coordinator.respec.mjs.map +0 -1
- package/src/sync/sync-local-spaces.respec.mts +0 -200
- package/src/sync/sync-peer/sync-peer-innerspace-v1.mts +0 -240
- package/src/sync/sync-saga-coordinator.respec.mts +0 -52
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { IbGib_V1, IbGibData_V1, IbGibRel8ns_V1 } from "@ibgib/ts-gib/dist/V1/types.mjs";
|
|
2
2
|
import { KeystoneIbGib_V1 } from "../../keystone/keystone-types.mjs";
|
|
3
3
|
import { SyncStage } from "../sync-constants.mjs";
|
|
4
|
-
import { SyncMode, SyncConflictStrategy,
|
|
4
|
+
import { SyncMode, SyncConflictStrategy, } from "../sync-types.mjs";
|
|
5
5
|
import { SYNC_SAGA_MSG_ATOM } from "./sync-saga-message-constants.mjs";
|
|
6
|
+
import { IbGibAddr } from "@ibgib/ts-gib/dist/types.mjs";
|
|
6
7
|
|
|
7
8
|
export interface SyncSagaMessageIb_V1 {
|
|
8
9
|
atom: typeof SYNC_SAGA_MSG_ATOM;
|
|
@@ -33,9 +34,66 @@ export interface SyncSagaMessageIbGib_V1 extends IbGib_V1<SyncSagaMessageData_V1
|
|
|
33
34
|
// CONCRETE MESSAGE PAYLOADS (EXTENDING DATA)
|
|
34
35
|
// ===========================================================================
|
|
35
36
|
|
|
36
|
-
|
|
37
|
+
/**
|
|
38
|
+
* The "Hello" of the sync protocol.
|
|
39
|
+
*
|
|
40
|
+
* Exchanges knowledge vectors (what I have) and identity (who I am).
|
|
41
|
+
*/
|
|
42
|
+
export interface SyncSagaMessageInitData_V1 extends SyncSagaMessageData_V1 {
|
|
37
43
|
stage: typeof SyncStage.init;
|
|
38
44
|
stones: string[];
|
|
45
|
+
/**
|
|
46
|
+
* Map of TJP (Temporal Joint Point) to latest known ibGibAddr.
|
|
47
|
+
* Use this to calculate what is missing (Delta).
|
|
48
|
+
*
|
|
49
|
+
* IOW, this is a map of the starting and end points of an ibGib's timeline.
|
|
50
|
+
*/
|
|
51
|
+
knowledgeVector: { [tjp: string]: IbGibAddr };
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* The Keystone Identity of the sender.
|
|
55
|
+
* Required for the first handshake or if the receiver doesn't know the sender.
|
|
56
|
+
*/
|
|
57
|
+
identity?: KeystoneIbGib_V1;
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* What the sender wants to do.
|
|
61
|
+
* e.g. "push" (I have data for you) or "pull" (Give me data).
|
|
62
|
+
* @default 'push'
|
|
63
|
+
*/
|
|
64
|
+
mode?: SyncMode;
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* This is also in the sync saga ibgib itself now...
|
|
68
|
+
*
|
|
69
|
+
* @see {@link SyncOptions.conflictStrategy}
|
|
70
|
+
*/
|
|
71
|
+
conflictStrategy?: SyncConflictStrategy;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export interface SyncSagaConflictInfo {
|
|
75
|
+
tjpAddr: string;
|
|
76
|
+
/**
|
|
77
|
+
* "local" is relative to the POV of the creator of the conflict info
|
|
78
|
+
*/
|
|
79
|
+
localAddr: string;
|
|
80
|
+
/**
|
|
81
|
+
* "remote" is relative to the POV of the creator of the conflict info
|
|
82
|
+
*/
|
|
83
|
+
remoteAddr: string;
|
|
84
|
+
/**
|
|
85
|
+
* Full history of the timeline from the Receiver's perspective.
|
|
86
|
+
* Used by Sender to find LCA.
|
|
87
|
+
*/
|
|
88
|
+
timelineAddrs: string[];
|
|
89
|
+
/**
|
|
90
|
+
* string description
|
|
91
|
+
*/
|
|
92
|
+
reason: string;
|
|
93
|
+
/**
|
|
94
|
+
* if true, this conflict should abort the sync
|
|
95
|
+
*/
|
|
96
|
+
terminal: boolean;
|
|
39
97
|
}
|
|
40
98
|
|
|
41
99
|
export interface SyncSagaMessageAckData_V1 extends SyncSagaMessageData_V1 {
|
|
@@ -43,31 +101,37 @@ export interface SyncSagaMessageAckData_V1 extends SyncSagaMessageData_V1 {
|
|
|
43
101
|
deltaReqAddrs: string[];
|
|
44
102
|
pushOfferAddrs: string[];
|
|
45
103
|
/**
|
|
46
|
-
* Map of group keys (TJP
|
|
47
|
-
*
|
|
104
|
+
* Map of group keys (TJP addr/constant addr) to list of known addresses
|
|
105
|
+
* in that timeline.
|
|
106
|
+
*
|
|
107
|
+
* This will be used by Sender to calculate differential payloads. So
|
|
108
|
+
* when the sender goes to get the dependency graph of its tip ibgib,
|
|
109
|
+
* it can skip all of the addrs in this.
|
|
110
|
+
*
|
|
111
|
+
* For example, say sender has A^5 and receiver only has A^3. Sender
|
|
112
|
+
* will see in this map { [A^0]: [A^0, A^1, A^2, A^3] } and know that
|
|
113
|
+
* it needs all of the dependencies of A^5 (including A^4), but it does
|
|
114
|
+
* NOT need to send A^0-A^3 (and dependencies) because receiver already has
|
|
115
|
+
* these.
|
|
48
116
|
*/
|
|
49
|
-
knowledgeVector?: { [
|
|
117
|
+
knowledgeVector?: { [tjpAddr: string]: string[] };
|
|
50
118
|
/**
|
|
51
119
|
* List of identified conflicts.
|
|
52
120
|
*
|
|
53
|
-
* If present, the Sender should use the `timelineAddrs` (Receiver's
|
|
54
|
-
* to compute the LCA and delta requirements for merging.
|
|
55
|
-
*/
|
|
56
|
-
conflicts?:
|
|
57
|
-
tjpAddr: string;
|
|
58
|
-
localAddr: string;
|
|
59
|
-
remoteAddr: string;
|
|
60
|
-
/**
|
|
61
|
-
* Full history of the timeline from the Receiver's perspective.
|
|
62
|
-
* Used by Sender to find LCA.
|
|
63
|
-
*/
|
|
64
|
-
timelineAddrs: string[];
|
|
65
|
-
reason: string;
|
|
66
|
-
terminal: boolean;
|
|
67
|
-
}[];
|
|
121
|
+
* If present, the Sender should use the `timelineAddrs` (Receiver's
|
|
122
|
+
* history) to compute the LCA and delta requirements for merging.
|
|
123
|
+
*/
|
|
124
|
+
conflicts?: SyncSagaConflictInfo[];
|
|
68
125
|
}
|
|
69
126
|
|
|
70
|
-
|
|
127
|
+
/**
|
|
128
|
+
* The "Body" of the sync protocol. Carries the manifests of data to be
|
|
129
|
+
* transferred.
|
|
130
|
+
*
|
|
131
|
+
* If any {@link payloadAddrsDomain} exist, then these will be populated in the
|
|
132
|
+
* {@link SyncSagaContextIbGib_V1}.
|
|
133
|
+
*/
|
|
134
|
+
export interface SyncSagaMessageDeltaData_V1 extends SyncSagaMessageData_V1 {
|
|
71
135
|
stage: typeof SyncStage.delta;
|
|
72
136
|
requests?: string[];
|
|
73
137
|
/**
|
|
@@ -75,9 +139,29 @@ export interface SyncSagaMessageDeltaData_V1 extends SyncSagaMessageData_V1, Syn
|
|
|
75
139
|
* and is ready to commit the transaction.
|
|
76
140
|
*/
|
|
77
141
|
proposeCommit?: boolean;
|
|
142
|
+
/**
|
|
143
|
+
* Domain ibgib addresses (actual user data being synced).
|
|
144
|
+
* These go to tempSpace ONLY until final commit.
|
|
145
|
+
*/
|
|
146
|
+
payloadAddrsDomain?: string[];
|
|
78
147
|
}
|
|
79
148
|
|
|
80
|
-
|
|
149
|
+
/**
|
|
150
|
+
* The "Seal" of the sync protocol.
|
|
151
|
+
*/
|
|
152
|
+
export interface SyncSagaMessageCommitData_V1 extends SyncSagaMessageData_V1 {
|
|
81
153
|
stage: typeof SyncStage.commit;
|
|
154
|
+
/**
|
|
155
|
+
* True if the session completed successfully from the sender's perspective.
|
|
156
|
+
*/
|
|
157
|
+
success: boolean;
|
|
158
|
+
/**
|
|
159
|
+
* Optional verification metadata (e.g. success count).
|
|
160
|
+
*/
|
|
161
|
+
successCount?: number;
|
|
162
|
+
/**
|
|
163
|
+
* If failed, the reasons why.
|
|
164
|
+
*/
|
|
165
|
+
errors?: string[];
|
|
82
166
|
}
|
|
83
167
|
|
package/src/sync/sync-types.mts
CHANGED
|
@@ -1,10 +1,15 @@
|
|
|
1
|
+
import { IbGibAddr } from "@ibgib/ts-gib/dist/types.mjs";
|
|
1
2
|
import { IbGib_V1, IbGibData_V1, IbGibRel8ns_V1 } from "@ibgib/ts-gib/dist/V1/types.mjs";
|
|
2
3
|
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
4
|
+
import { SubjectWitness } from "../common/pubsub/subject/subject-types.mjs";
|
|
5
|
+
import { SyncSagaContextIbGib_V1 } from "./sync-saga-context/sync-saga-context-types.mjs";
|
|
6
|
+
import { KeystoneIbGib_V1, } from "../keystone/keystone-types.mjs";
|
|
7
|
+
import { SYNC_ATOM, SYNC_MSG_REL8N_NAME, } from "./sync-constants.mjs";
|
|
5
8
|
import { IbGibSpaceAny } from "../witness/space/space-base-v1.mjs";
|
|
6
9
|
import { MetaspaceService } from "../witness/space/metaspace/metaspace-types.mjs";
|
|
7
10
|
import { SyncPeerWitness } from "./sync-peer/sync-peer-types.mjs";
|
|
11
|
+
import { FlatIbGibGraph } from "../common/other/graph-types.mjs";
|
|
12
|
+
import { SyncSagaConflictInfo } from "./sync-saga-message/sync-saga-message-types.mjs";
|
|
8
13
|
|
|
9
14
|
|
|
10
15
|
// #region SyncMode
|
|
@@ -34,6 +39,9 @@ export type SyncConflictStrategy =
|
|
|
34
39
|
| typeof SYNC_CONFLICT_STRATEGY_ABORT
|
|
35
40
|
| typeof SYNC_CONFLICT_STRATEGY_OPTIMISTIC
|
|
36
41
|
| typeof SYNC_CONFLICT_STRATEGY_MANUAL;
|
|
42
|
+
/**
|
|
43
|
+
* @see {@link SyncOptions.conflictStrategy}
|
|
44
|
+
*/
|
|
37
45
|
export const SyncConflictStrategy = {
|
|
38
46
|
abort: SYNC_CONFLICT_STRATEGY_ABORT,
|
|
39
47
|
optimistic: SYNC_CONFLICT_STRATEGY_OPTIMISTIC,
|
|
@@ -45,6 +53,16 @@ export function isValidSyncConflictStrategy(strategy: string): strategy is SyncC
|
|
|
45
53
|
}
|
|
46
54
|
// #endregion SyncConflictStrategy
|
|
47
55
|
|
|
56
|
+
/**
|
|
57
|
+
* Result of handling a saga frame.
|
|
58
|
+
* Separates control and domain payloads.
|
|
59
|
+
*/
|
|
60
|
+
export interface HandleSagaFrameResult {
|
|
61
|
+
frame: SyncIbGib_V1;
|
|
62
|
+
payloadIbGibsDomain?: IbGib_V1[];
|
|
63
|
+
conflictInfos?: SyncSagaConflictInfo;
|
|
64
|
+
}
|
|
65
|
+
|
|
48
66
|
export interface SyncOptions {
|
|
49
67
|
/**
|
|
50
68
|
* The peer we are syncing with.
|
|
@@ -81,96 +99,71 @@ export interface SyncOptions {
|
|
|
81
99
|
*/
|
|
82
100
|
identitySecret?: string;
|
|
83
101
|
/**
|
|
84
|
-
* How to handle conflicts when both Source and Dest have diverged on the
|
|
102
|
+
* How to handle conflicts when both Source and Dest have diverged on the
|
|
103
|
+
* same timeline.
|
|
104
|
+
*
|
|
85
105
|
* @default 'abort'
|
|
86
106
|
*/
|
|
87
107
|
conflictStrategy?: SyncConflictStrategy;
|
|
88
108
|
/**
|
|
89
|
-
*
|
|
90
|
-
*
|
|
109
|
+
* If true, creates an ephemeral session identity for the sync process to
|
|
110
|
+
* secure the sync transaction itself.
|
|
111
|
+
*
|
|
91
112
|
* @default true
|
|
92
113
|
*/
|
|
93
114
|
useSessionIdentity?: boolean;
|
|
94
115
|
}
|
|
95
116
|
|
|
96
|
-
|
|
97
|
-
// PROTOCOL PAYLOADS
|
|
98
|
-
// ===========================================================================
|
|
99
|
-
|
|
100
|
-
/**
|
|
101
|
-
* The "Hello" of the sync protocol.
|
|
102
|
-
* Exchanges knowledge vectors (what I have) and identity (who I am).
|
|
103
|
-
*/
|
|
104
|
-
export interface SyncInitData {
|
|
117
|
+
export interface SyncSagaInfo {
|
|
105
118
|
/**
|
|
106
|
-
*
|
|
107
|
-
* Use this to calculate what is missing (Delta).
|
|
108
|
-
*
|
|
109
|
-
* IOW, this is a map of the starting and end points of an ibGib's timeline.
|
|
119
|
+
* The unique ID of the saga.
|
|
110
120
|
*/
|
|
111
|
-
|
|
121
|
+
sagaId: string;
|
|
112
122
|
|
|
113
123
|
/**
|
|
114
|
-
*
|
|
115
|
-
*
|
|
124
|
+
* Observable stream of context updates happening during the saga.
|
|
125
|
+
*
|
|
126
|
+
* Subscribe to this to receive real-time progress throughout the sync
|
|
127
|
+
* process, whenever the sync ibgib is evolved (including when receiving a
|
|
128
|
+
* response from the peer).
|
|
116
129
|
*/
|
|
117
|
-
|
|
130
|
+
updates$: SubjectWitness<SyncSagaContextIbGib_V1>;
|
|
118
131
|
|
|
119
132
|
/**
|
|
120
|
-
*
|
|
121
|
-
* e.g. "push" (I have data for you) or "pull" (Give me data).
|
|
122
|
-
* @default 'push'
|
|
133
|
+
* Promise that resolves when the saga completes successfully.
|
|
123
134
|
*/
|
|
124
|
-
|
|
125
|
-
conflictStrategy?: SyncConflictStrategy;
|
|
135
|
+
done: Promise<void>;
|
|
126
136
|
}
|
|
127
137
|
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
export interface SyncRequestData {
|
|
133
|
-
/**
|
|
134
|
-
* The ibGib addresses requested.
|
|
135
|
-
*/
|
|
136
|
-
addrs: string[];
|
|
138
|
+
export interface SyncSagaFrameDependencyGraph {
|
|
139
|
+
sagaIbGib: SyncIbGib_V1;
|
|
140
|
+
msgStones: IbGib_V1[];
|
|
141
|
+
identities: KeystoneIbGib_V1[];
|
|
137
142
|
}
|
|
138
143
|
|
|
139
144
|
/**
|
|
140
|
-
*
|
|
141
|
-
* Carries the manifests of data to be transferred.
|
|
145
|
+
* given some group of domain ibgibs, this is info extracted from them.
|
|
142
146
|
*/
|
|
143
|
-
export interface
|
|
147
|
+
export interface DomainIbGibAnalysisInfo {
|
|
144
148
|
/**
|
|
145
|
-
*
|
|
146
|
-
* The actual data is streamed/batched separately.
|
|
149
|
+
* flat array of stones (ibgibs without dna)
|
|
147
150
|
*/
|
|
148
|
-
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
/**
|
|
152
|
-
* The "Seal" of the sync protocol.
|
|
153
|
-
* The authority is derived from the `identity` relation on the Keystone Frame.
|
|
154
|
-
*/
|
|
155
|
-
export interface SyncCommitData {
|
|
151
|
+
stones: IbGib_V1[];
|
|
156
152
|
/**
|
|
157
|
-
*
|
|
153
|
+
* map of tjp -> sorted timeline ibgib
|
|
158
154
|
*/
|
|
159
|
-
|
|
155
|
+
timelinesMap: { [tjp: string]: IbGib_V1[] };
|
|
160
156
|
/**
|
|
161
|
-
*
|
|
157
|
+
* sorted by dependency/interdependency
|
|
162
158
|
*/
|
|
163
|
-
|
|
159
|
+
topologicallySortedTjpAddrs: IbGibAddr[];
|
|
164
160
|
/**
|
|
165
|
-
*
|
|
161
|
+
* full dependency graph of domain ibgibs
|
|
166
162
|
*/
|
|
167
|
-
|
|
163
|
+
fullGraph: FlatIbGibGraph;
|
|
168
164
|
}
|
|
169
165
|
|
|
170
|
-
//
|
|
171
|
-
// IBGIB DATA STRUCTURES
|
|
172
|
-
// ===========================================================================
|
|
173
|
-
|
|
166
|
+
// #region Sync Ib, Data, Rel8ns, IbGib
|
|
174
167
|
export interface SyncIb_V1 {
|
|
175
168
|
atom: typeof SYNC_ATOM;
|
|
176
169
|
uuid: string;
|
|
@@ -204,11 +197,6 @@ export interface SyncData_V1 extends IbGibData_V1 {
|
|
|
204
197
|
* 2. Rate limiting / Backpressure (Don't process this old request).
|
|
205
198
|
*/
|
|
206
199
|
expirationTimestamp?: string;
|
|
207
|
-
|
|
208
|
-
/**
|
|
209
|
-
* Polymorphic payload based on stage.
|
|
210
|
-
*/
|
|
211
|
-
payload?: SyncInitData | SyncRequestData | SyncDeltaData | SyncCommitData;
|
|
212
200
|
}
|
|
213
201
|
|
|
214
202
|
export interface SyncRel8ns_V1 extends IbGibRel8ns_V1 {
|
|
@@ -217,30 +205,14 @@ export interface SyncRel8ns_V1 extends IbGibRel8ns_V1 {
|
|
|
217
205
|
* This MUST point to the specific Keystone Frame that authorizes this sync frame.
|
|
218
206
|
*/
|
|
219
207
|
identity?: string[];
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
export interface SyncIbGib_V1 extends IbGib_V1<SyncData_V1, SyncRel8ns_V1> { }
|
|
223
|
-
|
|
224
|
-
/**
|
|
225
|
-
* Information returned when starting a Sync Saga.
|
|
226
|
-
* Allows the caller to subscribe to updates and await completion.
|
|
227
|
-
*/
|
|
228
|
-
import { SubjectWitness } from "../common/pubsub/subject/subject-types.mjs";
|
|
229
|
-
import { SyncSagaContextIbGib_V1 } from "./sync-saga-context/sync-saga-context-types.mjs";
|
|
230
208
|
|
|
231
|
-
export interface SyncSagaInfo {
|
|
232
|
-
/**
|
|
233
|
-
* The unique ID of the saga.
|
|
234
|
-
*/
|
|
235
|
-
sagaId: string;
|
|
236
209
|
/**
|
|
237
|
-
*
|
|
238
|
-
*
|
|
210
|
+
* The message stone that contains the information about the particular
|
|
211
|
+
* stage of the sync process we are in.
|
|
239
212
|
*/
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
/**
|
|
243
|
-
* Promise that resolves when the saga completes successfully.
|
|
244
|
-
*/
|
|
245
|
-
done: Promise<void>;
|
|
213
|
+
[SYNC_MSG_REL8N_NAME]: IbGibAddr[];
|
|
246
214
|
}
|
|
215
|
+
|
|
216
|
+
export interface SyncIbGib_V1 extends IbGib_V1<SyncData_V1, SyncRel8ns_V1> { }
|
|
217
|
+
|
|
218
|
+
// #endregion Sync Ib, Data, Rel8ns, IbGib
|
|
@@ -27,6 +27,8 @@ import { SpecialIbGibType } from "../common/other/other-types.mjs";
|
|
|
27
27
|
import { getSpecialConfigKey, toDto } from "../common/other/ibgib-helper.mjs";
|
|
28
28
|
import { execInSpaceWithLocking, getFromSpace, persistTransformResult } from "../witness/space/space-helper.mjs";
|
|
29
29
|
import { TimelineHistoryInfo } from "./timeline-types.mjs";
|
|
30
|
+
import { toFlatGraph } from "../common/other/graph-helper.mjs";
|
|
31
|
+
import { FlatIbGibGraph } from "../common/other/graph-types.mjs";
|
|
30
32
|
|
|
31
33
|
const logalot = GLOBAL_LOG_A_LOT;
|
|
32
34
|
|
|
@@ -845,7 +847,16 @@ export async function createTimeline<TData extends IbGibData_V1 = IbGibData_V1>(
|
|
|
845
847
|
dna: noDna ? false : true,
|
|
846
848
|
nCounter: true,
|
|
847
849
|
squash: noDna ? true : false,
|
|
848
|
-
tjp: {
|
|
850
|
+
tjp: {
|
|
851
|
+
/**
|
|
852
|
+
* always use the timestamp to generate a unique timeline
|
|
853
|
+
*/
|
|
854
|
+
timestamp: true,
|
|
855
|
+
/**
|
|
856
|
+
* only generate new uuid if it's not in the incoming data
|
|
857
|
+
*/
|
|
858
|
+
uuid: !data?.uuid,
|
|
859
|
+
},
|
|
849
860
|
});
|
|
850
861
|
|
|
851
862
|
// Persist the new timeline and its DNA to the designated space.
|
|
@@ -938,21 +949,17 @@ export async function getHistory({
|
|
|
938
949
|
space ??= await metaspace.getLocalUserSpace({ lock: false });
|
|
939
950
|
if (!space) { throw new Error(`(UNEXPECTED) space is required. (E: e3d1c0b1a9a8b7c6d5e4f3a2b1a0b9c8)`); }
|
|
940
951
|
|
|
941
|
-
const ibGibsToMap = (ibgibs: IbGib_V1[]) => {
|
|
942
|
-
const map: { [addr: IbGibAddr]: IbGib_V1 } = {};
|
|
943
|
-
ibgibs.forEach(x => { map[getIbGibAddr({ ibGib: x })] = x; });
|
|
944
|
-
return map;
|
|
945
|
-
}
|
|
946
|
-
|
|
947
952
|
const headIbGib = getLatest ?
|
|
948
953
|
await getLatestTimelineIbGibDto_nonLocking({ timeline, metaspace, space }) :
|
|
949
954
|
toDto({ ibGib: timeline }) as IbGib_V1;
|
|
950
955
|
const headAddr = getIbGibAddr({ ibGib: headIbGib });
|
|
951
956
|
|
|
952
|
-
let pastIbGibMap:
|
|
957
|
+
let pastIbGibMap: FlatIbGibGraph = {};
|
|
953
958
|
|
|
954
959
|
const dnaMap = includeDna ?
|
|
955
|
-
|
|
960
|
+
toFlatGraph({
|
|
961
|
+
ibGibs: await getDna({ ibGib: headIbGib, metaspace, space })
|
|
962
|
+
}) :
|
|
956
963
|
undefined;
|
|
957
964
|
|
|
958
965
|
const pastAddrs = timeline.rel8ns?.past ?? [];
|
|
@@ -971,7 +978,7 @@ export async function getHistory({
|
|
|
971
978
|
throw new Error(`Could not get all dna ibGibs for ${headAddr}. Addrs not found: ${notFound.join(', ')}. Errors: ${resGet.errorMsg ?? '[unknown error (E: 887339d8b5580ed1ac9892a554255525)]'} (E: 4ab7a8a58adff4e238171f7c110c6825)`);
|
|
972
979
|
}
|
|
973
980
|
|
|
974
|
-
pastIbGibMap =
|
|
981
|
+
pastIbGibMap = toFlatGraph({ ibGibs: resGet.ibGibs! }) ?? {};
|
|
975
982
|
|
|
976
983
|
const orderedPastIbGibs: IbGib_V1[] = [];
|
|
977
984
|
for (const pastAddr of pastAddrs) {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { clone, getUUID, pretty } from '@ibgib/helper-gib/dist/helpers/utils-helper.mjs';
|
|
2
|
-
import { IbGib_V1, IbGibRel8ns_V1, } from '@ibgib/ts-gib/dist/V1/index.mjs';
|
|
2
|
+
import { getGib, IbGib_V1, IbGibRel8ns_V1, } from '@ibgib/ts-gib/dist/V1/index.mjs';
|
|
3
3
|
|
|
4
4
|
import { GLOBAL_LOG_A_LOT } from '../core-constants.mjs';
|
|
5
5
|
import { WitnessData_V1, Witness_V1, } from './witness-types.mjs';
|
|
@@ -90,6 +90,7 @@ export abstract class LightWitnessBase_V1<
|
|
|
90
90
|
*/
|
|
91
91
|
protected async initialize(): Promise<void> {
|
|
92
92
|
this.instanceId = await getUUID();
|
|
93
|
+
this.gib = await getGib({ ibGib: this.toIbGibDto() });
|
|
93
94
|
}
|
|
94
95
|
|
|
95
96
|
/**
|
|
@@ -199,7 +199,7 @@ export class InnerSpace_V1<
|
|
|
199
199
|
|
|
200
200
|
if (addrsAlreadyHave.length > 0) {
|
|
201
201
|
resultData.addrsAlreadyHave = addrsAlreadyHave;
|
|
202
|
-
resultData.warnings = (resultData.warnings || []).concat([`${lc} already had addr(s): ${addrsAlreadyHave.join('|')}
|
|
202
|
+
resultData.warnings = (resultData.warnings || []).concat([`${lc} already had addr(s): ${addrsAlreadyHave.join('|')}`, '(W: 7120c8267c2aaf8cad5d2f3850666826)']);
|
|
203
203
|
}
|
|
204
204
|
resultData.success = true;
|
|
205
205
|
} catch (error) {
|
package/test_output.log
CHANGED
|
Binary file
|
package/tmp.md
CHANGED
|
@@ -1,44 +1,62 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
1
|
+
Great! Ty for going through that! In crafting my response for you, I have since updated the libs\core-gib\src\sync\README.md file, so please re-read that file for me. There were several inaccuracies and it is much improved.
|
|
2
|
+
|
|
3
|
+
Now for your questions. Let's get to them!
|
|
4
|
+
|
|
5
|
+
## foundational
|
|
6
|
+
|
|
7
|
+
1. I don't know if the conflict merging aspect is working well or not. I have implemented a similar optimistic strategy before in a previous sync prototype that worked well (though that implementation was very slow). This approach is the same, except two additions: a) naive text merge if the transform targets `data.text`, and b) adding "graft info" (libs\core-gib\src\sync\graft-info) to the timeline after the merge so that original divergent states aren't lost. Both of these should be straightforward though. The main problem right now is trying to isolate the various stages of the "ping-pong" exchange.
|
|
8
|
+
|
|
9
|
+
2. Tjp is fully implemented and has been for several years now. Including the tjpGib in the full address, instead of just using a bare hash, has proved immensely useful (and as you know, this is completely novel/innovative as there is no other protocol that does this).
|
|
10
|
+
|
|
11
|
+
3. We originally had control and domain payloads grouped together and we had to separate them. However, this was done poorly by the original agent, and may yet still be incomplete. AFAICT though, the most recent attempt at completing the separation was successful. The current blocking issue is that it's hard to tell what is causing the test in sync-conflict.respec.mts to fail. The logging gets too verbose and is hard to track throughout the entire transaction.
|
|
12
|
+
|
|
13
|
+
4. We are deferring integrating the session keystones until we get the non-keystone workflow going.
|
|
14
|
+
|
|
15
|
+
## current state
|
|
16
|
+
|
|
17
|
+
5. The immediate symptom is the test in `sync-conflict.respect.mts` is failing. You can see the output of the most recent run in C:\Users\billm\antigravity\ibgib\libs\core-gib\test_output.log . But it is a coarse test, and really, it's hard to even read the test itself. But it's also hard to try to create more granular unit tests, though that may be the right path forward.
|
|
18
|
+
|
|
19
|
+
6. AFAICT the v16 implementation plan has been partially implemented. For example, `handleAckFrame` in `saga-coordinator.mts` still has not had the `payloadIbGibs` separated properly, i.e., there is still a variable called `payloadIbGibs`, despite us mandating that there should be NO references to this. Every instance of that string should be separated into control and domain payload ibgibs. It's actually looking like that `handleAckFrame` is not right anymore, though the libs\core-gib\src\sync\sync-innerspace-dest-ahead.respec.mts test is still passing so it's not completely wrong. But if you read through that method, you'll see it's not right and there is a not-implemented error line in there.
|
|
20
|
+
|
|
21
|
+
The same is true for `payloadAddrs`, which should NOT exist anywhere. But it still is on `SyncDeltaData`. It should not be "deprecated", it should have been removed and references to it should have led us to more places to complete the separation of control and domain addrs.
|
|
22
|
+
|
|
23
|
+
7. I think part of the problem is that our implementation plans have been too complex and we didn't follow through to ensure that each and every piece was taken care of. Perhaps a todo/checklist would be more appropriate. But I devote quite a bit of time to refine the implementation plan, and then the nuances get lost. So we have a bunch of messy, half-completed code. Granted, this is a relatively complex sync algorithm, but it should absolutely be feasible.
|
|
24
|
+
|
|
25
|
+
But let me speak to your specific questions on this:
|
|
26
|
+
|
|
27
|
+
* Too many iterations/refactorings without clear direction?
|
|
28
|
+
* I don't mind the iterations. I mind thinking that something is complete and it is not. This is partially my fault in not breaking it down and unit testing it more granularly, but that would add a huge amount of overhead, and what inevitably happens is we troubleshoot the tests and not the code.
|
|
29
|
+
* Type boundaries bleeding between concerns?
|
|
30
|
+
* Our types are pretty well created by this point.
|
|
31
|
+
* The polling mechanism feeling hacky?
|
|
32
|
+
* No, the polling mechanism is fine.
|
|
33
|
+
* Race conditions or timing issues?
|
|
34
|
+
* There have been no race conditions that I know of.
|
|
35
|
+
|
|
36
|
+
I had previous success with Antigravity on other smaller tasks, but this one seems to have been a little too advanced for the Gemini 3 High model. Your model seems to be performing better, however we hit the smaller context window rather quickly. Overall, I am just trying not to have to go through every single LoC myself, though it's looking like that is more and more likely.
|
|
37
|
+
|
|
38
|
+
Here is another example I've come across that is messy. Look at the following code found in `sync-saga-coordinator.mts`:
|
|
39
|
+
|
|
40
|
+
```
|
|
41
|
+
// 4. EXECUTE SAGA LOOP (FSM)
|
|
42
|
+
// Inject tempSpace into peer so it can pull control payloads to the right place
|
|
43
|
+
if ('opts' in peer && peer.opts) {
|
|
44
|
+
(peer.opts as any).senderTempSpace = tempSpace;
|
|
45
|
+
}
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
This is terrible! This is a hack so that if we have a `SyncPeerInnerspace_V1`, we set its temp space. This is "messy" to say the least.
|
|
49
|
+
|
|
50
|
+
### sync test results
|
|
51
|
+
|
|
52
|
+
* libs\core-gib\src\sync\sync-innerspace.respec.mts PASSING
|
|
53
|
+
* libs\core-gib\src\sync\sync-innerspace-constants.respec.mts PASSING
|
|
54
|
+
* libs\core-gib\src\sync\sync-innerspace-dest-ahead.respec.mts PASSING
|
|
55
|
+
* libs\core-gib\src\sync\sync-innerspace-deep-updates.respec.mts PASSING
|
|
56
|
+
* libs\core-gib\src\sync\sync-innerspace-multiple-timelines.respec.mts PASSING
|
|
57
|
+
* libs\core-gib\src\sync\sync-innerspace-partial-update.respec.mts PASSING
|
|
58
|
+
* libs\core-gib\src\sync\sync-conflict.respec.mts FAILING
|
|
59
|
+
|
|
60
|
+
## how you can help
|
|
61
|
+
|
|
62
|
+
All of the simpler tests that don't require a Delta frame back and forth (short back-and-forth to resolve the conflict) are all passing. I _think_ that the merge logic itself is fine, though I haven't gone through that with a fine-tooth comb yet. To me, the most obvious problem right now is that the mechanism for passing domain ibgibs back and forth is not working. For example, look in libs\core-gib\src\sync\sync-saga-coordinator.mts for `srcGraph`. That was added early on in the code sketch. IIRC, it was supposed to be for the domain ibgibs in the original sender's location. Is that another name for payloadIbGibs, but stored in a map? How does it get populated? How do the payload ibgibs go from the peer to the handle____ methods? So don't do any coding right now. Perhaps this whole conversation will just be you and I talking about the code as I go through it. So just point me to things that you see and I will make changes, because I need to wrap this up in the next 3 days. Do note that I have made some changes since starting this message to you, and it doesn't build at the moment. But overall there are still things that you can just point me to if you just examine the logic.
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"sync-local-spaces.respec.d.mts","sourceRoot":"","sources":["../../src/sync/sync-local-spaces.respec.mts"],"names":[],"mappings":""}
|