@nahisaho/yata-scale 1.8.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -0
- package/dist/CacheManager.d.ts +107 -0
- package/dist/CacheManager.d.ts.map +1 -0
- package/dist/CacheManager.js +343 -0
- package/dist/CacheManager.js.map +1 -0
- package/dist/IndexManager.d.ts +103 -0
- package/dist/IndexManager.d.ts.map +1 -0
- package/dist/IndexManager.js +385 -0
- package/dist/IndexManager.js.map +1 -0
- package/dist/MigrationHelper.d.ts +91 -0
- package/dist/MigrationHelper.d.ts.map +1 -0
- package/dist/MigrationHelper.js +220 -0
- package/dist/MigrationHelper.js.map +1 -0
- package/dist/QueryCoordinator.d.ts +76 -0
- package/dist/QueryCoordinator.d.ts.map +1 -0
- package/dist/QueryCoordinator.js +298 -0
- package/dist/QueryCoordinator.js.map +1 -0
- package/dist/ShardManager.d.ts +48 -0
- package/dist/ShardManager.d.ts.map +1 -0
- package/dist/ShardManager.js +193 -0
- package/dist/ShardManager.js.map +1 -0
- package/dist/SyncController.d.ts +102 -0
- package/dist/SyncController.d.ts.map +1 -0
- package/dist/SyncController.js +286 -0
- package/dist/SyncController.js.map +1 -0
- package/dist/YataScaleManager.d.ts +75 -0
- package/dist/YataScaleManager.d.ts.map +1 -0
- package/dist/YataScaleManager.js +265 -0
- package/dist/YataScaleManager.js.map +1 -0
- package/dist/cache/CacheManager.d.ts +116 -0
- package/dist/cache/CacheManager.d.ts.map +1 -0
- package/dist/cache/CacheManager.js +286 -0
- package/dist/cache/CacheManager.js.map +1 -0
- package/dist/cache/InvalidationManager.d.ts +91 -0
- package/dist/cache/InvalidationManager.d.ts.map +1 -0
- package/dist/cache/InvalidationManager.js +155 -0
- package/dist/cache/InvalidationManager.js.map +1 -0
- package/dist/cache/L1Cache.d.ts +97 -0
- package/dist/cache/L1Cache.d.ts.map +1 -0
- package/dist/cache/L1Cache.js +225 -0
- package/dist/cache/L1Cache.js.map +1 -0
- package/dist/cache/L2Cache.d.ts +93 -0
- package/dist/cache/L2Cache.d.ts.map +1 -0
- package/dist/cache/L2Cache.js +248 -0
- package/dist/cache/L2Cache.js.map +1 -0
- package/dist/cache/L3Cache.d.ts +101 -0
- package/dist/cache/L3Cache.d.ts.map +1 -0
- package/dist/cache/L3Cache.js +229 -0
- package/dist/cache/L3Cache.js.map +1 -0
- package/dist/cache/index.d.ts +12 -0
- package/dist/cache/index.d.ts.map +1 -0
- package/dist/cache/index.js +11 -0
- package/dist/cache/index.js.map +1 -0
- package/dist/errors.d.ts +255 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +515 -0
- package/dist/errors.js.map +1 -0
- package/dist/index/BPlusTreeIndex.d.ts +101 -0
- package/dist/index/BPlusTreeIndex.d.ts.map +1 -0
- package/dist/index/BPlusTreeIndex.js +310 -0
- package/dist/index/BPlusTreeIndex.js.map +1 -0
- package/dist/index/BloomFilter.d.ts +85 -0
- package/dist/index/BloomFilter.d.ts.map +1 -0
- package/dist/index/BloomFilter.js +215 -0
- package/dist/index/BloomFilter.js.map +1 -0
- package/dist/index/FullTextIndex.d.ts +87 -0
- package/dist/index/FullTextIndex.d.ts.map +1 -0
- package/dist/index/FullTextIndex.js +213 -0
- package/dist/index/FullTextIndex.js.map +1 -0
- package/dist/index/GraphIndex.d.ts +116 -0
- package/dist/index/GraphIndex.d.ts.map +1 -0
- package/dist/index/GraphIndex.js +308 -0
- package/dist/index/GraphIndex.js.map +1 -0
- package/dist/index/IndexManager.d.ts +105 -0
- package/dist/index/IndexManager.d.ts.map +1 -0
- package/dist/index/IndexManager.js +287 -0
- package/dist/index/IndexManager.js.map +1 -0
- package/dist/index/index.d.ts +12 -0
- package/dist/index/index.d.ts.map +1 -0
- package/dist/index/index.js +11 -0
- package/dist/index/index.js.map +1 -0
- package/dist/index.d.ts +12 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +12 -0
- package/dist/index.js.map +1 -0
- package/dist/query/DistributedExecutor.d.ts +78 -0
- package/dist/query/DistributedExecutor.d.ts.map +1 -0
- package/dist/query/DistributedExecutor.js +232 -0
- package/dist/query/DistributedExecutor.js.map +1 -0
- package/dist/query/QueryCoordinator.d.ts +98 -0
- package/dist/query/QueryCoordinator.d.ts.map +1 -0
- package/dist/query/QueryCoordinator.js +246 -0
- package/dist/query/QueryCoordinator.js.map +1 -0
- package/dist/query/QueryPlanner.d.ts +88 -0
- package/dist/query/QueryPlanner.d.ts.map +1 -0
- package/dist/query/QueryPlanner.js +235 -0
- package/dist/query/QueryPlanner.js.map +1 -0
- package/dist/query/WorkerPool.d.ts +108 -0
- package/dist/query/WorkerPool.d.ts.map +1 -0
- package/dist/query/WorkerPool.js +195 -0
- package/dist/query/WorkerPool.js.map +1 -0
- package/dist/query/index.d.ts +11 -0
- package/dist/query/index.d.ts.map +1 -0
- package/dist/query/index.js +10 -0
- package/dist/query/index.js.map +1 -0
- package/dist/shard/GraphPartitionStrategy.d.ts +65 -0
- package/dist/shard/GraphPartitionStrategy.d.ts.map +1 -0
- package/dist/shard/GraphPartitionStrategy.js +208 -0
- package/dist/shard/GraphPartitionStrategy.js.map +1 -0
- package/dist/shard/HashPartitionStrategy.d.ts +56 -0
- package/dist/shard/HashPartitionStrategy.d.ts.map +1 -0
- package/dist/shard/HashPartitionStrategy.js +169 -0
- package/dist/shard/HashPartitionStrategy.js.map +1 -0
- package/dist/shard/RangePartitionStrategy.d.ts +63 -0
- package/dist/shard/RangePartitionStrategy.d.ts.map +1 -0
- package/dist/shard/RangePartitionStrategy.js +157 -0
- package/dist/shard/RangePartitionStrategy.js.map +1 -0
- package/dist/shard/Rebalancer.d.ts +70 -0
- package/dist/shard/Rebalancer.d.ts.map +1 -0
- package/dist/shard/Rebalancer.js +184 -0
- package/dist/shard/Rebalancer.js.map +1 -0
- package/dist/shard/ShardManager.d.ts +123 -0
- package/dist/shard/ShardManager.d.ts.map +1 -0
- package/dist/shard/ShardManager.js +353 -0
- package/dist/shard/ShardManager.js.map +1 -0
- package/dist/shard/ShardRouter.d.ts +102 -0
- package/dist/shard/ShardRouter.d.ts.map +1 -0
- package/dist/shard/ShardRouter.js +181 -0
- package/dist/shard/ShardRouter.js.map +1 -0
- package/dist/shard/index.d.ts +13 -0
- package/dist/shard/index.d.ts.map +1 -0
- package/dist/shard/index.js +12 -0
- package/dist/shard/index.js.map +1 -0
- package/dist/sync/ConflictResolver.d.ts +71 -0
- package/dist/sync/ConflictResolver.d.ts.map +1 -0
- package/dist/sync/ConflictResolver.js +214 -0
- package/dist/sync/ConflictResolver.js.map +1 -0
- package/dist/sync/SyncController.d.ts +150 -0
- package/dist/sync/SyncController.d.ts.map +1 -0
- package/dist/sync/SyncController.js +304 -0
- package/dist/sync/SyncController.js.map +1 -0
- package/dist/sync/VectorClock.d.ts +102 -0
- package/dist/sync/VectorClock.d.ts.map +1 -0
- package/dist/sync/VectorClock.js +189 -0
- package/dist/sync/VectorClock.js.map +1 -0
- package/dist/sync/WALManager.d.ts +120 -0
- package/dist/sync/WALManager.d.ts.map +1 -0
- package/dist/sync/WALManager.js +240 -0
- package/dist/sync/WALManager.js.map +1 -0
- package/dist/sync/index.d.ts +11 -0
- package/dist/sync/index.d.ts.map +1 -0
- package/dist/sync/index.js +10 -0
- package/dist/sync/index.js.map +1 -0
- package/dist/types.d.ts +323 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +7 -0
- package/dist/types.js.map +1 -0
- package/package.json +86 -0
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @nahisaho/yata-scale - Graph Partition Strategy
|
|
3
|
+
*
|
|
4
|
+
* Graph-based partitioning using connected components
|
|
5
|
+
*/
|
|
6
|
+
/**
|
|
7
|
+
* Graph partition strategy based on connected components
|
|
8
|
+
*/
|
|
9
|
+
export class GraphPartitionStrategy {
|
|
10
|
+
type = 'graph';
|
|
11
|
+
componentMap = new Map();
|
|
12
|
+
shardIds;
|
|
13
|
+
defaultShardId;
|
|
14
|
+
constructor(shardIds) {
|
|
15
|
+
if (shardIds.length === 0) {
|
|
16
|
+
throw new Error('At least one shard is required');
|
|
17
|
+
}
|
|
18
|
+
this.shardIds = [...shardIds];
|
|
19
|
+
this.defaultShardId = shardIds[0];
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Initialize the partition map from entities and relationships
|
|
23
|
+
*/
|
|
24
|
+
async initialize(entities, relationships) {
|
|
25
|
+
// Build adjacency list
|
|
26
|
+
const adjacency = new Map();
|
|
27
|
+
for (const entity of entities) {
|
|
28
|
+
if (!adjacency.has(entity.id)) {
|
|
29
|
+
adjacency.set(entity.id, new Set());
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
for (const rel of relationships) {
|
|
33
|
+
if (!adjacency.has(rel.sourceId)) {
|
|
34
|
+
adjacency.set(rel.sourceId, new Set());
|
|
35
|
+
}
|
|
36
|
+
if (!adjacency.has(rel.targetId)) {
|
|
37
|
+
adjacency.set(rel.targetId, new Set());
|
|
38
|
+
}
|
|
39
|
+
adjacency.get(rel.sourceId).add(rel.targetId);
|
|
40
|
+
adjacency.get(rel.targetId).add(rel.sourceId);
|
|
41
|
+
}
|
|
42
|
+
// Find connected components using Union-Find
|
|
43
|
+
const components = this.findConnectedComponents(adjacency);
|
|
44
|
+
// Assign components to shards (round-robin by component size)
|
|
45
|
+
this.assignComponentsToShards(components);
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Find connected components using Union-Find
|
|
49
|
+
*/
|
|
50
|
+
findConnectedComponents(adjacency) {
|
|
51
|
+
const parent = new Map();
|
|
52
|
+
const rank = new Map();
|
|
53
|
+
// Initialize Union-Find
|
|
54
|
+
for (const id of adjacency.keys()) {
|
|
55
|
+
parent.set(id, id);
|
|
56
|
+
rank.set(id, 0);
|
|
57
|
+
}
|
|
58
|
+
// Find with path compression
|
|
59
|
+
const find = (x) => {
|
|
60
|
+
if (parent.get(x) !== x) {
|
|
61
|
+
parent.set(x, find(parent.get(x)));
|
|
62
|
+
}
|
|
63
|
+
return parent.get(x);
|
|
64
|
+
};
|
|
65
|
+
// Union by rank
|
|
66
|
+
const union = (x, y) => {
|
|
67
|
+
const rootX = find(x);
|
|
68
|
+
const rootY = find(y);
|
|
69
|
+
if (rootX !== rootY) {
|
|
70
|
+
const rankX = rank.get(rootX);
|
|
71
|
+
const rankY = rank.get(rootY);
|
|
72
|
+
if (rankX < rankY) {
|
|
73
|
+
parent.set(rootX, rootY);
|
|
74
|
+
}
|
|
75
|
+
else if (rankX > rankY) {
|
|
76
|
+
parent.set(rootY, rootX);
|
|
77
|
+
}
|
|
78
|
+
else {
|
|
79
|
+
parent.set(rootY, rootX);
|
|
80
|
+
rank.set(rootX, rankX + 1);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
// Union connected nodes
|
|
85
|
+
for (const [node, neighbors] of adjacency) {
|
|
86
|
+
for (const neighbor of neighbors) {
|
|
87
|
+
union(node, neighbor);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
// Group by component
|
|
91
|
+
const components = new Map();
|
|
92
|
+
for (const id of adjacency.keys()) {
|
|
93
|
+
const root = find(id);
|
|
94
|
+
if (!components.has(root)) {
|
|
95
|
+
components.set(root, new Set());
|
|
96
|
+
}
|
|
97
|
+
components.get(root).add(id);
|
|
98
|
+
}
|
|
99
|
+
return components;
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Assign components to shards using best-fit decreasing
|
|
103
|
+
*/
|
|
104
|
+
assignComponentsToShards(components) {
|
|
105
|
+
// Sort components by size (descending)
|
|
106
|
+
const sortedComponents = [...components.entries()].sort((a, b) => b[1].size - a[1].size);
|
|
107
|
+
// Track shard sizes
|
|
108
|
+
const shardSizes = new Map();
|
|
109
|
+
for (const shardId of this.shardIds) {
|
|
110
|
+
shardSizes.set(shardId, 0);
|
|
111
|
+
}
|
|
112
|
+
// Assign each component to the least loaded shard
|
|
113
|
+
for (const [_componentId, members] of sortedComponents) {
|
|
114
|
+
// Find shard with minimum load
|
|
115
|
+
let minShardId = this.shardIds[0];
|
|
116
|
+
let minSize = Infinity;
|
|
117
|
+
for (const [shardId, size] of shardSizes) {
|
|
118
|
+
if (size < minSize) {
|
|
119
|
+
minSize = size;
|
|
120
|
+
minShardId = shardId;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
// Assign all members to this shard
|
|
124
|
+
for (const memberId of members) {
|
|
125
|
+
this.componentMap.set(memberId, minShardId);
|
|
126
|
+
}
|
|
127
|
+
// Update shard size
|
|
128
|
+
shardSizes.set(minShardId, minSize + members.size);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Get shard ID for an entity
|
|
133
|
+
*/
|
|
134
|
+
getShardId(entityId, _metadata) {
|
|
135
|
+
return this.componentMap.get(entityId) ?? this.defaultShardId;
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Get shard IDs for multiple entities
|
|
139
|
+
*/
|
|
140
|
+
getShardIds(entityIds) {
|
|
141
|
+
const result = new Map();
|
|
142
|
+
for (const entityId of entityIds) {
|
|
143
|
+
result.set(entityId, this.getShardId(entityId));
|
|
144
|
+
}
|
|
145
|
+
return result;
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Calculate remapping (graph partitioning requires recomputation)
|
|
149
|
+
*/
|
|
150
|
+
remap(_oldShardCount, _newShardCount) {
|
|
151
|
+
// Graph-based partitioning typically requires full recomputation
|
|
152
|
+
// Return empty map - caller should reinitialize
|
|
153
|
+
return new Map();
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Add entity to a specific shard
|
|
157
|
+
*/
|
|
158
|
+
assignEntity(entityId, shardId) {
|
|
159
|
+
if (!this.shardIds.includes(shardId)) {
|
|
160
|
+
throw new Error(`Unknown shard: ${shardId}`);
|
|
161
|
+
}
|
|
162
|
+
this.componentMap.set(entityId, shardId);
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* Add entity near related entity (locality)
|
|
166
|
+
*/
|
|
167
|
+
assignEntityNear(entityId, relatedEntityId) {
|
|
168
|
+
const shardId = this.componentMap.get(relatedEntityId);
|
|
169
|
+
if (shardId) {
|
|
170
|
+
this.componentMap.set(entityId, shardId);
|
|
171
|
+
}
|
|
172
|
+
else {
|
|
173
|
+
this.componentMap.set(entityId, this.defaultShardId);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* Get shard distribution statistics
|
|
178
|
+
*/
|
|
179
|
+
getDistribution() {
|
|
180
|
+
const distribution = new Map();
|
|
181
|
+
for (const shardId of this.shardIds) {
|
|
182
|
+
distribution.set(shardId, 0);
|
|
183
|
+
}
|
|
184
|
+
for (const shardId of this.componentMap.values()) {
|
|
185
|
+
distribution.set(shardId, (distribution.get(shardId) ?? 0) + 1);
|
|
186
|
+
}
|
|
187
|
+
return distribution;
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* Get all shard IDs
|
|
191
|
+
*/
|
|
192
|
+
getShardIdsList() {
|
|
193
|
+
return [...this.shardIds];
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Get shard count
|
|
197
|
+
*/
|
|
198
|
+
get shardCount() {
|
|
199
|
+
return this.shardIds.length;
|
|
200
|
+
}
|
|
201
|
+
/**
|
|
202
|
+
* Get entity count
|
|
203
|
+
*/
|
|
204
|
+
get entityCount() {
|
|
205
|
+
return this.componentMap.size;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
//# sourceMappingURL=GraphPartitionStrategy.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"GraphPartitionStrategy.js","sourceRoot":"","sources":["../../src/shard/GraphPartitionStrategy.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAUH;;GAEG;AACH,MAAM,OAAO,sBAAsB;IACjB,IAAI,GAA0B,OAAO,CAAC;IAC9C,YAAY,GAAwB,IAAI,GAAG,EAAE,CAAC;IAC9C,QAAQ,CAAW;IACnB,cAAc,CAAS;IAE/B,YAAY,QAAkB;QAC5B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QACpD,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC;QAC9B,IAAI,CAAC,cAAc,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CACd,QAAkB,EAClB,aAA6B;QAE7B,uBAAuB;QACvB,MAAM,SAAS,GAAG,IAAI,GAAG,EAAuB,CAAC;QAEjD,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;YAC9B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC9B,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QAED,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;YAChC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACjC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;YACzC,CAAC;YACD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACjC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;YACzC,CAAC;YACD,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC/C,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACjD,CAAC;QAED,6CAA6C;QAC7C,MAAM,UAAU,GAAG,IAAI,CAAC,uBAAuB,CAAC,SAAS,CAAC,CAAC;QAE3D,8DAA8D;QAC9D,IAAI,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IAED;;OAEG;IACK,uBAAuB,CAC7B,SAAmC;QAEnC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;QACzC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;QAEvC,wBAAwB;QACxB,KAAK,MAAM,EAAE,IAAI,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC;YAClC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;YACnB,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,6BAA6B;QAC7B,MAAM,IAAI,GAAG,CAAC,CAAS,EAAU,EAAE;YACjC,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;gBACxB,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC;YACtC,CAAC;YACD,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC;QACxB,CAAC,CAAC;QAEF,gBAAgB;QAChB,MAAM,KAAK,GAAG,CAAC,CAAS,EAAE,CAAS,EAAQ,EAAE;YAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;YACtB,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;YAEtB,IAAI,KAAK,KAAK,KAAK,EAAE,CAAC;gBACpB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC;gBAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC;gBAE/B,IAAI,KAAK,GAAG,KAAK,EAAE,CAAC;oBAClB,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;gBAC3B,CAAC;qBAAM,IAAI,KAAK,GAAG,KAAK,EAAE,CAAC;oBACzB,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;gBAC3B,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;oBACzB,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;gBAC7B,CAAC;YACH,CAAC;QACH,CAAC,CAAC;QAEF,wBAAwB;QACxB,KAAK,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,SAAS,EAAE,CAAC;YAC1C,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;gBACjC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QAED,qBAAqB;QACrB,MAAM,UAAU,GAAG,IAAI,GAAG,EAAuB,CAAC;QAClD,KAAK,MAAM,EAAE,IAAI,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;YACtB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC1B,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;YAClC,CAAC;YACD,UAAU,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChC,CAAC;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAED;;OAEG;IACK,wBAAwB,CAAC,UAAoC;QACnE,uCAAuC;QACvC,MAAM,gBAAgB,GAAG,CAAC,GAAG,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CACrD,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAChC,CAAC;QAEF,oBAAoB;QACpB,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC7C,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpC,UAAU,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAC7B,CAAC;QAED,kDAAkD;QAClD,KAAK,MAAM,CAAC,YAAY,EAAE,OAAO,CAAC,IAAI,gBAAgB,EAAE,CAAC;YACvD,+BAA+B;YAC/B,IAAI,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YAClC,IAAI,OAAO,GAAG,QAAQ,CAAC;YAEvB,KAAK,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,UAAU,EAAE,CAAC;gBACzC,IAAI,IAAI,GAAG,OAAO,EAAE,CAAC;oBACnB,OAAO,GAAG,IAAI,CAAC;oBACf,UAAU,GAAG,OAAO,CAAC;gBACvB,CAAC;YACH,CAAC;YAED,mCAAmC;YACnC,KAAK,MAAM,QAAQ,IAAI,OAAO,EAAE,CAAC;gBAC/B,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;YAC9C,CAAC;YAED,oBAAoB;YACpB,UAAU,CAAC,GAAG,CAAC,UAAU,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,QAAgB,EAAE,SAAmC;QAC9D,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,cAAc,CAAC;IAChE,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,SAAmB;QAC7B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;QACzC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC;QAClD,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAsB,EAAE,cAAsB;QAClD,iEAAiE;QACjE,gDAAgD;QAChD,OAAO,IAAI,GAAG,EAAE,CAAC;IACnB,CAAC;IAED;;OAEG;IACH,YAAY,CAAC,QAAgB,EAAE,OAAe;QAC5C,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,kBAAkB,OAAO,EAAE,CAAC,CAAC;QAC/C,CAAC;QACD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,gBAAgB,CAAC,QAAgB,EAAE,eAAuB;QACxD,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QACvD,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC3C,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;IAED;;OAEG;IACH,eAAe;QACb,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;QAE/C,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAC/B,CAAC;QAED,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC;YACjD,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAClE,CAAC;QAED,OAAO,YAAY,CAAC;IACtB,CAAC;IAED;;OAEG;IACH,eAAe;QACb,OAAO,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC;IAChC,CAAC;CACF"}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @nahisaho/yata-scale - Hash Partition Strategy
|
|
3
|
+
*
|
|
4
|
+
* Consistent hash-based partitioning for even distribution
|
|
5
|
+
*/
|
|
6
|
+
import type { PartitionStrategy, PartitionStrategyType, EntityMetadata } from '../types.js';
|
|
7
|
+
/**
|
|
8
|
+
* Hash partition strategy using consistent hashing
|
|
9
|
+
*/
|
|
10
|
+
export declare class HashPartitionStrategy implements PartitionStrategy {
|
|
11
|
+
private readonly shardIds;
|
|
12
|
+
readonly type: PartitionStrategyType;
|
|
13
|
+
private readonly virtualNodes;
|
|
14
|
+
private ring;
|
|
15
|
+
private sortedHashes;
|
|
16
|
+
constructor(shardIds: string[], options?: {
|
|
17
|
+
virtualNodes?: number;
|
|
18
|
+
});
|
|
19
|
+
/**
|
|
20
|
+
* Build the consistent hash ring
|
|
21
|
+
*/
|
|
22
|
+
private buildRing;
|
|
23
|
+
/**
|
|
24
|
+
* Simple hash function (FNV-1a variant)
|
|
25
|
+
*/
|
|
26
|
+
private hash;
|
|
27
|
+
/**
|
|
28
|
+
* Find the shard for a given hash using binary search
|
|
29
|
+
*/
|
|
30
|
+
private findShard;
|
|
31
|
+
/**
|
|
32
|
+
* Get shard ID for an entity
|
|
33
|
+
*/
|
|
34
|
+
getShardId(entityId: string, _metadata?: Partial<EntityMetadata>): string;
|
|
35
|
+
/**
|
|
36
|
+
* Calculate remapping when shard count changes
|
|
37
|
+
*/
|
|
38
|
+
remap(oldShardCount: number, newShardCount: number): Map<string, string>;
|
|
39
|
+
/**
|
|
40
|
+
* Find shard excluding certain shards
|
|
41
|
+
*/
|
|
42
|
+
private findShardExcluding;
|
|
43
|
+
/**
|
|
44
|
+
* Add a shard to the ring
|
|
45
|
+
*/
|
|
46
|
+
addShard(shardId: string): void;
|
|
47
|
+
/**
|
|
48
|
+
* Remove a shard from the ring
|
|
49
|
+
*/
|
|
50
|
+
removeShard(shardId: string): void;
|
|
51
|
+
/**
|
|
52
|
+
* Get shard count
|
|
53
|
+
*/
|
|
54
|
+
get shardCount(): number;
|
|
55
|
+
}
|
|
56
|
+
//# sourceMappingURL=HashPartitionStrategy.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"HashPartitionStrategy.d.ts","sourceRoot":"","sources":["../../src/shard/HashPartitionStrategy.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EACV,iBAAiB,EACjB,qBAAqB,EACrB,cAAc,EACf,MAAM,aAAa,CAAC;AAErB;;GAEG;AACH,qBAAa,qBAAsB,YAAW,iBAAiB;IAO3D,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAN3B,SAAgB,IAAI,EAAE,qBAAqB,CAAU;IACrD,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,IAAI,CAAkC;IAC9C,OAAO,CAAC,YAAY,CAAgB;gBAGjB,QAAQ,EAAE,MAAM,EAAE,EACnC,OAAO,GAAE;QAAE,YAAY,CAAC,EAAE,MAAM,CAAA;KAAO;IAMzC;;OAEG;IACH,OAAO,CAAC,SAAS;IAejB;;OAEG;IACH,OAAO,CAAC,IAAI;IASZ;;OAEG;IACH,OAAO,CAAC,SAAS;IAuBjB;;OAEG;IACH,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,GAAG,MAAM;IAgBzE;;OAEG;IACH,KAAK,CAAC,aAAa,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC;IAuBxE;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAyB1B;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAQ/B;;OAEG;IACH,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAgBlC;;OAEG;IACH,IAAI,UAAU,IAAI,MAAM,CAEvB;CACF"}
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @nahisaho/yata-scale - Hash Partition Strategy
|
|
3
|
+
*
|
|
4
|
+
* Consistent hash-based partitioning for even distribution
|
|
5
|
+
*/
|
|
6
|
+
/**
|
|
7
|
+
* Hash partition strategy using consistent hashing
|
|
8
|
+
*/
|
|
9
|
+
export class HashPartitionStrategy {
|
|
10
|
+
shardIds;
|
|
11
|
+
type = 'hash';
|
|
12
|
+
virtualNodes;
|
|
13
|
+
ring = new Map();
|
|
14
|
+
sortedHashes = [];
|
|
15
|
+
constructor(shardIds, options = {}) {
|
|
16
|
+
this.shardIds = shardIds;
|
|
17
|
+
this.virtualNodes = options.virtualNodes ?? 150;
|
|
18
|
+
this.buildRing();
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Build the consistent hash ring
|
|
22
|
+
*/
|
|
23
|
+
buildRing() {
|
|
24
|
+
this.ring.clear();
|
|
25
|
+
this.sortedHashes = [];
|
|
26
|
+
for (const shardId of this.shardIds) {
|
|
27
|
+
for (let i = 0; i < this.virtualNodes; i++) {
|
|
28
|
+
const hash = this.hash(`${shardId}:${i}`);
|
|
29
|
+
this.ring.set(hash, shardId);
|
|
30
|
+
this.sortedHashes.push(hash);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
this.sortedHashes.sort((a, b) => a - b);
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Simple hash function (FNV-1a variant)
|
|
37
|
+
*/
|
|
38
|
+
hash(key) {
|
|
39
|
+
let hash = 2166136261;
|
|
40
|
+
for (let i = 0; i < key.length; i++) {
|
|
41
|
+
hash ^= key.charCodeAt(i);
|
|
42
|
+
hash = Math.imul(hash, 16777619);
|
|
43
|
+
}
|
|
44
|
+
return hash >>> 0;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Find the shard for a given hash using binary search
|
|
48
|
+
*/
|
|
49
|
+
findShard(hash) {
|
|
50
|
+
if (this.sortedHashes.length === 0) {
|
|
51
|
+
throw new Error('No shards configured');
|
|
52
|
+
}
|
|
53
|
+
// Binary search for the first hash >= target
|
|
54
|
+
let left = 0;
|
|
55
|
+
let right = this.sortedHashes.length;
|
|
56
|
+
while (left < right) {
|
|
57
|
+
const mid = (left + right) >>> 1;
|
|
58
|
+
if (this.sortedHashes[mid] < hash) {
|
|
59
|
+
left = mid + 1;
|
|
60
|
+
}
|
|
61
|
+
else {
|
|
62
|
+
right = mid;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
// Wrap around if we've gone past the end
|
|
66
|
+
const index = left >= this.sortedHashes.length ? 0 : left;
|
|
67
|
+
return this.ring.get(this.sortedHashes[index]);
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Get shard ID for an entity
|
|
71
|
+
*/
|
|
72
|
+
getShardId(entityId, _metadata) {
|
|
73
|
+
const hash = this.hash(entityId);
|
|
74
|
+
return this.findShard(hash);
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Get shard IDs for multiple entities
|
|
78
|
+
*/
|
|
79
|
+
getShardIds(entityIds) {
|
|
80
|
+
const result = new Map();
|
|
81
|
+
for (const entityId of entityIds) {
|
|
82
|
+
result.set(entityId, this.getShardId(entityId));
|
|
83
|
+
}
|
|
84
|
+
return result;
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Calculate remapping when shard count changes
|
|
88
|
+
*/
|
|
89
|
+
remap(oldShardCount, newShardCount) {
|
|
90
|
+
// For consistent hashing, remapping is minimal
|
|
91
|
+
// Only keys that were on removed shards need to move
|
|
92
|
+
const remapping = new Map();
|
|
93
|
+
if (newShardCount > oldShardCount) {
|
|
94
|
+
// Adding shards - some keys will move to new shards
|
|
95
|
+
// This is handled by the ring naturally when rebuilt
|
|
96
|
+
}
|
|
97
|
+
else if (newShardCount < oldShardCount) {
|
|
98
|
+
// Removing shards - keys from removed shards need reassignment
|
|
99
|
+
const removedShards = this.shardIds.slice(newShardCount);
|
|
100
|
+
for (const [hash, shardId] of this.ring) {
|
|
101
|
+
if (removedShards.includes(shardId)) {
|
|
102
|
+
// Find new shard for this hash
|
|
103
|
+
const newShard = this.findShardExcluding(hash, removedShards);
|
|
104
|
+
remapping.set(String(hash), newShard);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
return remapping;
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Find shard excluding certain shards
|
|
112
|
+
*/
|
|
113
|
+
findShardExcluding(hash, excludeShards) {
|
|
114
|
+
let left = 0;
|
|
115
|
+
let right = this.sortedHashes.length;
|
|
116
|
+
while (left < right) {
|
|
117
|
+
const mid = (left + right) >>> 1;
|
|
118
|
+
if (this.sortedHashes[mid] < hash) {
|
|
119
|
+
left = mid + 1;
|
|
120
|
+
}
|
|
121
|
+
else {
|
|
122
|
+
right = mid;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
// Find first non-excluded shard
|
|
126
|
+
for (let i = 0; i < this.sortedHashes.length; i++) {
|
|
127
|
+
const index = (left + i) % this.sortedHashes.length;
|
|
128
|
+
const shard = this.ring.get(this.sortedHashes[index]);
|
|
129
|
+
if (!excludeShards.includes(shard)) {
|
|
130
|
+
return shard;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
throw new Error('No available shards after exclusion');
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Add a shard to the ring
|
|
137
|
+
*/
|
|
138
|
+
addShard(shardId) {
|
|
139
|
+
if (this.shardIds.includes(shardId)) {
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
142
|
+
this.shardIds.push(shardId);
|
|
143
|
+
this.buildRing();
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Remove a shard from the ring
|
|
147
|
+
*/
|
|
148
|
+
removeShard(shardId) {
|
|
149
|
+
const index = this.shardIds.indexOf(shardId);
|
|
150
|
+
if (index === -1) {
|
|
151
|
+
return;
|
|
152
|
+
}
|
|
153
|
+
this.shardIds.splice(index, 1);
|
|
154
|
+
this.buildRing();
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Get all shard IDs
|
|
158
|
+
*/
|
|
159
|
+
getShardIds() {
|
|
160
|
+
return [...this.shardIds];
|
|
161
|
+
}
|
|
162
|
+
/**
|
|
163
|
+
* Get shard count
|
|
164
|
+
*/
|
|
165
|
+
get shardCount() {
|
|
166
|
+
return this.shardIds.length;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
//# sourceMappingURL=HashPartitionStrategy.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"HashPartitionStrategy.js","sourceRoot":"","sources":["../../src/shard/HashPartitionStrategy.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAQH;;GAEG;AACH,MAAM,OAAO,qBAAqB;IAOb;IANH,IAAI,GAA0B,MAAM,CAAC;IACpC,YAAY,CAAS;IAC9B,IAAI,GAAwB,IAAI,GAAG,EAAE,CAAC;IACtC,YAAY,GAAa,EAAE,CAAC;IAEpC,YACmB,QAAkB,EACnC,UAAqC,EAAE;QADtB,aAAQ,GAAR,QAAQ,CAAU;QAGnC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,GAAG,CAAC;QAChD,IAAI,CAAC,SAAS,EAAE,CAAC;IACnB,CAAC;IAED;;OAEG;IACK,SAAS;QACf,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAClB,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;QAEvB,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,OAAO,IAAI,CAAC,EAAE,CAAC,CAAC;gBAC1C,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;gBAC7B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QAED,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1C,CAAC;IAED;;OAEG;IACK,IAAI,CAAC,GAAW;QACtB,IAAI,IAAI,GAAG,UAAU,CAAC;QACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,IAAI,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAC1B,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACnC,CAAC;QACD,OAAO,IAAI,KAAK,CAAC,CAAC;IACpB,CAAC;IAED;;OAEG;IACK,SAAS,CAAC,IAAY;QAC5B,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;QAC1C,CAAC;QAED,6CAA6C;QAC7C,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,IAAI,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;QAErC,OAAO,IAAI,GAAG,KAAK,EAAE,CAAC;YACpB,MAAM,GAAG,GAAG,CAAC,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YACjC,IAAI,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,IAAI,EAAE,CAAC;gBAClC,IAAI,GAAG,GAAG,GAAG,CAAC,CAAC;YACjB,CAAC;iBAAM,CAAC;gBACN,KAAK,GAAG,GAAG,CAAC;YACd,CAAC;QACH,CAAC;QAED,yCAAyC;QACzC,MAAM,KAAK,GAAG,IAAI,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC1D,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAE,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,QAAgB,EAAE,SAAmC;QAC9D,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACjC,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,SAAmB;QAC7B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;QACzC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC;QAClD,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAqB,EAAE,aAAqB;QAChD,+CAA+C;QAC/C,qDAAqD;QACrD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;QAE5C,IAAI,aAAa,GAAG,aAAa,EAAE,CAAC;YAClC,oDAAoD;YACpD,qDAAqD;QACvD,CAAC;aAAM,IAAI,aAAa,GAAG,aAAa,EAAE,CAAC;YACzC,+DAA+D;YAC/D,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YACzD,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBACxC,IAAI,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;oBACpC,+BAA+B;oBAC/B,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;oBAC9D,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,CAAC;gBACxC,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACK,kBAAkB,CAAC,IAAY,EAAE,aAAuB;QAC9D,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,IAAI,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;QAErC,OAAO,IAAI,GAAG,KAAK,EAAE,CAAC;YACpB,MAAM,GAAG,GAAG,CAAC,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YACjC,IAAI,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,IAAI,EAAE,CAAC;gBAClC,IAAI,GAAG,GAAG,GAAG,CAAC,CAAC;YACjB,CAAC;iBAAM,CAAC;gBACN,KAAK,GAAG,GAAG,CAAC;YACd,CAAC;QACH,CAAC;QAED,gCAAgC;QAChC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAClD,MAAM,KAAK,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;YACpD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAE,CAAC;YACvD,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBACnC,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED;;OAEG;IACH,QAAQ,CAAC,OAAe;QACtB,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YACpC,OAAO;QACT,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC5B,IAAI,CAAC,SAAS,EAAE,CAAC;IACnB,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,OAAe;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC7C,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAC/B,IAAI,CAAC,SAAS,EAAE,CAAC;IACnB,CAAC;IAED;;OAEG;IACH,WAAW;QACT,OAAO,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;IAC9B,CAAC;CACF"}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @nahisaho/yata-scale - Range Partition Strategy
|
|
3
|
+
*
|
|
4
|
+
* Range-based partitioning for ordered data distribution
|
|
5
|
+
*/
|
|
6
|
+
import type { PartitionStrategy, PartitionStrategyType, EntityMetadata } from '../types.js';
|
|
7
|
+
/**
|
|
8
|
+
* Range configuration for a shard
|
|
9
|
+
*/
|
|
10
|
+
export interface RangeConfig {
|
|
11
|
+
readonly shardId: string;
|
|
12
|
+
readonly start: string;
|
|
13
|
+
readonly end: string;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Range partition strategy for ordered key distribution
|
|
17
|
+
*/
|
|
18
|
+
export declare class RangePartitionStrategy implements PartitionStrategy {
|
|
19
|
+
readonly type: PartitionStrategyType;
|
|
20
|
+
private ranges;
|
|
21
|
+
constructor(ranges: RangeConfig[]);
|
|
22
|
+
/**
|
|
23
|
+
* Validate ranges don't overlap and sort them
|
|
24
|
+
*/
|
|
25
|
+
private validateAndSortRanges;
|
|
26
|
+
/**
|
|
27
|
+
* Extract partition key from entity ID or metadata
|
|
28
|
+
*/
|
|
29
|
+
private extractPartitionKey;
|
|
30
|
+
/**
|
|
31
|
+
* Get shard ID for an entity using binary search
|
|
32
|
+
*/
|
|
33
|
+
getShardId(entityId: string, metadata?: Partial<EntityMetadata>): string;
|
|
34
|
+
/**
|
|
35
|
+
* Get shard IDs for multiple entities
|
|
36
|
+
*/
|
|
37
|
+
getShardIds(entityIds: string[]): Map<string, string>;
|
|
38
|
+
/**
|
|
39
|
+
* Calculate remapping when ranges change
|
|
40
|
+
*/
|
|
41
|
+
remap(oldShardCount: number, newShardCount: number): Map<string, string>;
|
|
42
|
+
/**
|
|
43
|
+
* Split a range into two
|
|
44
|
+
*/
|
|
45
|
+
splitRange(shardId: string, splitPoint: string, newShardId: string): RangeConfig[];
|
|
46
|
+
/**
|
|
47
|
+
* Merge two adjacent ranges
|
|
48
|
+
*/
|
|
49
|
+
mergeRanges(shardId1: string, shardId2: string, mergedShardId: string): RangeConfig;
|
|
50
|
+
/**
|
|
51
|
+
* Get all ranges
|
|
52
|
+
*/
|
|
53
|
+
getRanges(): readonly RangeConfig[];
|
|
54
|
+
/**
|
|
55
|
+
* Get all shard IDs
|
|
56
|
+
*/
|
|
57
|
+
getShardIdsList(): string[];
|
|
58
|
+
/**
|
|
59
|
+
* Get shard count
|
|
60
|
+
*/
|
|
61
|
+
get shardCount(): number;
|
|
62
|
+
}
|
|
63
|
+
//# sourceMappingURL=RangePartitionStrategy.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"RangePartitionStrategy.d.ts","sourceRoot":"","sources":["../../src/shard/RangePartitionStrategy.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EACV,iBAAiB,EACjB,qBAAqB,EACrB,cAAc,EACf,MAAM,aAAa,CAAC;AAErB;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,qBAAa,sBAAuB,YAAW,iBAAiB;IAC9D,SAAgB,IAAI,EAAE,qBAAqB,CAAW;IACtD,OAAO,CAAC,MAAM,CAAgB;gBAElB,MAAM,EAAE,WAAW,EAAE;IAIjC;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAoB7B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAW3B;;OAEG;IACH,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,CAAC,cAAc,CAAC,GAAG,MAAM;IA6BxE;;OAEG;IACH,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC;IAQrD;;OAEG;IACH,KAAK,CAAC,aAAa,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC;IAaxE;;OAEG;IACH,UAAU,CACR,OAAO,EAAE,MAAM,EACf,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,GACjB,WAAW,EAAE;IAuBhB;;OAEG;IACH,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,WAAW;IA8BnF;;OAEG;IACH,SAAS,IAAI,SAAS,WAAW,EAAE;IAInC;;OAEG;IACH,eAAe,IAAI,MAAM,EAAE;IAI3B;;OAEG;IACH,IAAI,UAAU,IAAI,MAAM,CAEvB;CACF"}
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @nahisaho/yata-scale - Range Partition Strategy
|
|
3
|
+
*
|
|
4
|
+
* Range-based partitioning for ordered data distribution
|
|
5
|
+
*/
|
|
6
|
+
/**
|
|
7
|
+
* Range partition strategy for ordered key distribution
|
|
8
|
+
*/
|
|
9
|
+
export class RangePartitionStrategy {
|
|
10
|
+
type = 'range';
|
|
11
|
+
ranges;
|
|
12
|
+
constructor(ranges) {
|
|
13
|
+
this.ranges = this.validateAndSortRanges(ranges);
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Validate ranges don't overlap and sort them
|
|
17
|
+
*/
|
|
18
|
+
validateAndSortRanges(ranges) {
|
|
19
|
+
if (ranges.length === 0) {
|
|
20
|
+
throw new Error('At least one range is required');
|
|
21
|
+
}
|
|
22
|
+
// Sort by start key
|
|
23
|
+
const sorted = [...ranges].sort((a, b) => a.start.localeCompare(b.start));
|
|
24
|
+
// Check for overlaps
|
|
25
|
+
for (let i = 0; i < sorted.length - 1; i++) {
|
|
26
|
+
if (sorted[i].end > sorted[i + 1].start) {
|
|
27
|
+
throw new Error(`Range overlap detected between ${sorted[i].shardId} and ${sorted[i + 1].shardId}`);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
return sorted;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Extract partition key from entity ID or metadata
|
|
34
|
+
*/
|
|
35
|
+
extractPartitionKey(entityId, metadata) {
|
|
36
|
+
// If namespace is available, use namespace:id format
|
|
37
|
+
if (metadata?.source) {
|
|
38
|
+
return `${metadata.source}:${entityId}`;
|
|
39
|
+
}
|
|
40
|
+
return entityId;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Get shard ID for an entity using binary search
|
|
44
|
+
*/
|
|
45
|
+
getShardId(entityId, metadata) {
|
|
46
|
+
const key = this.extractPartitionKey(entityId, metadata);
|
|
47
|
+
// Binary search for the correct range
|
|
48
|
+
let left = 0;
|
|
49
|
+
let right = this.ranges.length - 1;
|
|
50
|
+
while (left <= right) {
|
|
51
|
+
const mid = (left + right) >>> 1;
|
|
52
|
+
const range = this.ranges[mid];
|
|
53
|
+
if (key < range.start) {
|
|
54
|
+
right = mid - 1;
|
|
55
|
+
}
|
|
56
|
+
else if (key >= range.end) {
|
|
57
|
+
left = mid + 1;
|
|
58
|
+
}
|
|
59
|
+
else {
|
|
60
|
+
return range.shardId;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
// If key is before first range, use first shard
|
|
64
|
+
if (key < this.ranges[0].start) {
|
|
65
|
+
return this.ranges[0].shardId;
|
|
66
|
+
}
|
|
67
|
+
// If key is after last range, use last shard
|
|
68
|
+
return this.ranges[this.ranges.length - 1].shardId;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Get shard IDs for multiple entities
|
|
72
|
+
*/
|
|
73
|
+
getShardIds(entityIds) {
|
|
74
|
+
const result = new Map();
|
|
75
|
+
for (const entityId of entityIds) {
|
|
76
|
+
result.set(entityId, this.getShardId(entityId));
|
|
77
|
+
}
|
|
78
|
+
return result;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Calculate remapping when ranges change
|
|
82
|
+
*/
|
|
83
|
+
remap(oldShardCount, newShardCount) {
|
|
84
|
+
const remapping = new Map();
|
|
85
|
+
// Range-based partitioning requires explicit range reconfiguration
|
|
86
|
+
// This method returns the keys that would need to move
|
|
87
|
+
if (newShardCount !== oldShardCount) {
|
|
88
|
+
// In range partitioning, remapping depends on how ranges are split/merged
|
|
89
|
+
// This is a placeholder - actual implementation depends on split strategy
|
|
90
|
+
}
|
|
91
|
+
return remapping;
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Split a range into two
|
|
95
|
+
*/
|
|
96
|
+
splitRange(shardId, splitPoint, newShardId) {
|
|
97
|
+
const index = this.ranges.findIndex((r) => r.shardId === shardId);
|
|
98
|
+
if (index === -1) {
|
|
99
|
+
throw new Error(`Shard not found: ${shardId}`);
|
|
100
|
+
}
|
|
101
|
+
const range = this.ranges[index];
|
|
102
|
+
if (splitPoint <= range.start || splitPoint >= range.end) {
|
|
103
|
+
throw new Error(`Split point ${splitPoint} is outside range [${range.start}, ${range.end})`);
|
|
104
|
+
}
|
|
105
|
+
const newRanges = [
|
|
106
|
+
{ shardId: range.shardId, start: range.start, end: splitPoint },
|
|
107
|
+
{ shardId: newShardId, start: splitPoint, end: range.end },
|
|
108
|
+
];
|
|
109
|
+
// Replace old range with new ranges
|
|
110
|
+
this.ranges.splice(index, 1, ...newRanges);
|
|
111
|
+
return newRanges;
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Merge two adjacent ranges
|
|
115
|
+
*/
|
|
116
|
+
mergeRanges(shardId1, shardId2, mergedShardId) {
|
|
117
|
+
const index1 = this.ranges.findIndex((r) => r.shardId === shardId1);
|
|
118
|
+
const index2 = this.ranges.findIndex((r) => r.shardId === shardId2);
|
|
119
|
+
if (index1 === -1 || index2 === -1) {
|
|
120
|
+
throw new Error('One or both shards not found');
|
|
121
|
+
}
|
|
122
|
+
// Ensure they are adjacent
|
|
123
|
+
if (Math.abs(index1 - index2) !== 1) {
|
|
124
|
+
throw new Error('Ranges must be adjacent to merge');
|
|
125
|
+
}
|
|
126
|
+
const [firstIndex, secondIndex] = index1 < index2 ? [index1, index2] : [index2, index1];
|
|
127
|
+
const firstRange = this.ranges[firstIndex];
|
|
128
|
+
const secondRange = this.ranges[secondIndex];
|
|
129
|
+
const mergedRange = {
|
|
130
|
+
shardId: mergedShardId,
|
|
131
|
+
start: firstRange.start,
|
|
132
|
+
end: secondRange.end,
|
|
133
|
+
};
|
|
134
|
+
// Replace both ranges with merged range
|
|
135
|
+
this.ranges.splice(firstIndex, 2, mergedRange);
|
|
136
|
+
return mergedRange;
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Get all ranges
|
|
140
|
+
*/
|
|
141
|
+
getRanges() {
|
|
142
|
+
return this.ranges;
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Get all shard IDs
|
|
146
|
+
*/
|
|
147
|
+
getShardIdsList() {
|
|
148
|
+
return this.ranges.map((r) => r.shardId);
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Get shard count
|
|
152
|
+
*/
|
|
153
|
+
get shardCount() {
|
|
154
|
+
return this.ranges.length;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
//# sourceMappingURL=RangePartitionStrategy.js.map
|