@earth-app/collegedb 1.0.2 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +154 -7
- package/dist/index.js +7 -7
- package/dist/index.js.map +5 -5
- package/dist/kvmap.d.ts +80 -26
- package/dist/kvmap.d.ts.map +1 -1
- package/dist/types.d.ts +24 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -189,6 +189,96 @@ This approach provides:
|
|
|
189
189
|
- **Optimal read performance**: Queries use `hash` strategy for consistent, high-performance routing
|
|
190
190
|
- **Flexibility**: Each operation type can use the most appropriate routing strategy
|
|
191
191
|
|
|
192
|
+
## Multi-Key Shard Mappings
|
|
193
|
+
|
|
194
|
+
CollegeDB supports **multiple lookup keys** for the same record, allowing you to query by username, email, ID, or any unique identifier. Keys are automatically hashed with SHA-256 for security and privacy.
|
|
195
|
+
|
|
196
|
+
```typescript
|
|
197
|
+
import { collegedb, first, run, KVShardMapper } from 'collegedb';
|
|
198
|
+
|
|
199
|
+
collegedb(
|
|
200
|
+
{
|
|
201
|
+
kv: env.KV,
|
|
202
|
+
shards: { 'db-east': env.DB_EAST, 'db-west': env.DB_WEST },
|
|
203
|
+
hashShardMappings: true, // Default: enabled for security
|
|
204
|
+
strategy: 'hash'
|
|
205
|
+
},
|
|
206
|
+
async () => {
|
|
207
|
+
// Create a user with multiple lookup keys
|
|
208
|
+
const mapper = new KVShardMapper(env.KV, { hashShardMappings: true });
|
|
209
|
+
|
|
210
|
+
await mapper.setShardMapping('user-123', 'db-east', ['username:john_doe', 'email:john@example.com', 'id:123']);
|
|
211
|
+
|
|
212
|
+
// Now you can query by ANY of these keys
|
|
213
|
+
const byId = await first('user-123', 'SELECT * FROM users WHERE id = ?', ['user-123']);
|
|
214
|
+
const byUsername = await first('username:john_doe', 'SELECT * FROM users WHERE username = ?', ['john_doe']);
|
|
215
|
+
const byEmail = await first('email:john@example.com', 'SELECT * FROM users WHERE email = ?', ['john@example.com']);
|
|
216
|
+
|
|
217
|
+
// All queries route to the same shard (db-east)
|
|
218
|
+
console.log('All queries find the same user:', byId?.name);
|
|
219
|
+
}
|
|
220
|
+
);
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
### Adding Lookup Keys to Existing Mappings
|
|
224
|
+
|
|
225
|
+
s
|
|
226
|
+
|
|
227
|
+
```typescript
|
|
228
|
+
const mapper = new KVShardMapper(env.KV);
|
|
229
|
+
|
|
230
|
+
// User initially created with just ID
|
|
231
|
+
await mapper.setShardMapping('user-456', 'db-west');
|
|
232
|
+
|
|
233
|
+
// Later, add additional lookup methods
|
|
234
|
+
await mapper.addLookupKeys('user-456', ['email:jane@example.com', 'username:jane']);
|
|
235
|
+
|
|
236
|
+
// Now works with any key
|
|
237
|
+
const user = await first('email:jane@example.com', 'SELECT * FROM users WHERE email = ?', ['jane@example.com']);
|
|
238
|
+
```
|
|
239
|
+
|
|
240
|
+
### Security and Privacy
|
|
241
|
+
|
|
242
|
+
**SHA-256 Hashing (Enabled by Default)**: Sensitive data like emails are hashed before being stored as KV keys, protecting user privacy:
|
|
243
|
+
|
|
244
|
+
```typescript
|
|
245
|
+
// With hashShardMappings: true (default)
|
|
246
|
+
// KV stores: "shard:a1b2c3d4..." instead of "shard:email:user@example.com"
|
|
247
|
+
|
|
248
|
+
const config = {
|
|
249
|
+
kv: env.KV,
|
|
250
|
+
shards: {
|
|
251
|
+
/* ... */
|
|
252
|
+
},
|
|
253
|
+
hashShardMappings: true, // Hashes keys with SHA-256
|
|
254
|
+
strategy: 'hash'
|
|
255
|
+
};
|
|
256
|
+
```
|
|
257
|
+
|
|
258
|
+
**⚠️ Performance Trade-off**: When hashing is enabled, operations like `getKeysForShard()` cannot return original key names, only hashed versions. For full key recovery, disable hashing:
|
|
259
|
+
|
|
260
|
+
```typescript
|
|
261
|
+
const config = {
|
|
262
|
+
hashShardMappings: false // Disables hashing - keys stored in plain text
|
|
263
|
+
};
|
|
264
|
+
```
|
|
265
|
+
|
|
266
|
+
### Multi-Key Management
|
|
267
|
+
|
|
268
|
+
```typescript
|
|
269
|
+
const mapper = new KVShardMapper(env.KV);
|
|
270
|
+
|
|
271
|
+
// Get all lookup keys for a mapping
|
|
272
|
+
const allKeys = await mapper.getAllLookupKeys('email:user@example.com');
|
|
273
|
+
console.log(allKeys); // ['user-123', 'username:john', 'email:user@example.com']
|
|
274
|
+
|
|
275
|
+
// Update shard assignment (updates all keys)
|
|
276
|
+
await mapper.updateShardMapping('username:john', 'db-central');
|
|
277
|
+
|
|
278
|
+
// Delete mapping (removes all associated keys)
|
|
279
|
+
await mapper.deleteShardMapping('user-123');
|
|
280
|
+
```
|
|
281
|
+
|
|
192
282
|
## Drop-in Replacement for Existing Databases
|
|
193
283
|
|
|
194
284
|
CollegeDB supports **seamless, automatic integration** with existing D1 databases that already contain data. Simply add your existing databases as shards in the configuration. CollegeDB will automatically detect existing data and create the necessary shard mappings **without requiring any manual migration steps**.
|
|
@@ -557,10 +647,13 @@ interface CollegeDBConfig {
|
|
|
557
647
|
strategy?: ShardingStrategy | MixedShardingStrategy;
|
|
558
648
|
targetRegion?: D1Region;
|
|
559
649
|
shardLocations?: Record<string, ShardLocation>;
|
|
560
|
-
disableAutoMigration?: boolean;
|
|
650
|
+
disableAutoMigration?: boolean; // Default: false
|
|
651
|
+
hashShardMappings?: boolean; // Default: true
|
|
561
652
|
}
|
|
562
653
|
```
|
|
563
654
|
|
|
655
|
+
When `hashShardMappings` is enabled (default), original keys cannot be recovered during shard operations like `getKeysForShard()`. This is intentional for privacy but means you'll get fewer results from such operations. For full key recovery, set `hashShardMappings: false`, but be aware this may expose sensitive data in KV keys.
|
|
656
|
+
|
|
564
657
|
#### Strategy Types
|
|
565
658
|
|
|
566
659
|
```typescript
|
|
@@ -1176,7 +1269,7 @@ _SELECT, VALUES, TABLE, PRAGMA, ..._
|
|
|
1176
1269
|
| CollegeDB (100 shards) | ~60-90ms | 100x parallel capacity | ~75-80x |
|
|
1177
1270
|
| CollegeDB (1000 shards) | ~65-95ms | 1000x parallel capacity | ~650-700x |
|
|
1178
1271
|
|
|
1179
|
-
\*Includes KV lookup overhead (~5-15ms)
|
|
1272
|
+
\*Includes KV lookup overhead (~5-15ms) and SHA-256 hashing overhead (~1-3ms when `hashShardMappings: true`)
|
|
1180
1273
|
|
|
1181
1274
|
#### Write Performance
|
|
1182
1275
|
|
|
@@ -1189,14 +1282,14 @@ _INSERT, UPDATE, DELETE, ..._
|
|
|
1189
1282
|
| CollegeDB (100 shards) | ~95-145ms | ~4,200 writes/sec | ~84x |
|
|
1190
1283
|
| CollegeDB (1000 shards) | ~105-160ms | ~35,000 writes/sec | ~700x |
|
|
1191
1284
|
|
|
1192
|
-
\*Includes KV mapping creation/update overhead (~10-25ms)
|
|
1285
|
+
\*Includes KV mapping creation/update overhead (~10-25ms) and SHA-256 hashing overhead (~1-3ms when `hashShardMappings: true`)
|
|
1193
1286
|
|
|
1194
1287
|
### Strategy-Specific Performance
|
|
1195
1288
|
|
|
1196
1289
|
#### Hash Strategy
|
|
1197
1290
|
|
|
1198
1291
|
- **Best for**: Consistent performance, even data distribution
|
|
1199
|
-
- **Latency**: Lowest overhead (no coordinator calls)
|
|
1292
|
+
- **Latency**: Lowest overhead (no coordinator calls, ~1-3ms SHA-256 hashing when enabled)
|
|
1200
1293
|
- **Throughput**: Optimal for high-volume scenarios
|
|
1201
1294
|
|
|
1202
1295
|
| Shards | Avg Latency | Distribution Quality | Coordinator Dependency |
|
|
@@ -1323,7 +1416,7 @@ _INSERT, UPDATE, DELETE, ..._
|
|
|
1323
1416
|
// Recommended: Hash reads + Location writes
|
|
1324
1417
|
{
|
|
1325
1418
|
strategy: { read: 'hash', write: 'location' },
|
|
1326
|
-
targetRegion:
|
|
1419
|
+
targetRegion: getClosestRegionFromIP(request), // Dynamic region targeting
|
|
1327
1420
|
shardLocations: {
|
|
1328
1421
|
'db-americas': { region: 'wnam', priority: 2 },
|
|
1329
1422
|
'db-europe': { region: 'weur', priority: 2 },
|
|
@@ -1430,6 +1523,58 @@ _INSERT, UPDATE, DELETE, ..._
|
|
|
1430
1523
|
| **Balanced** | 50/50 | `{read: 'hash', write: 'hash'}` | Consistent performance |
|
|
1431
1524
|
| **Analytics** | 95% reads | `{read: 'location', write: 'round-robin'}` | Regional + perfect distribution |
|
|
1432
1525
|
|
|
1526
|
+
### SHA-256 Hashing Performance Impact
|
|
1527
|
+
|
|
1528
|
+
CollegeDB uses SHA-256 hashing by default (`hashShardMappings: true`) to protect sensitive data in KV keys. This adds a small but measurable performance overhead:
|
|
1529
|
+
|
|
1530
|
+
#### Hashing Performance Characteristics
|
|
1531
|
+
|
|
1532
|
+
| Operation Type | SHA-256 Overhead | Total Latency Impact | Security Benefit |
|
|
1533
|
+
| ------------------ | ---------------- | -------------------- | ---------------------------- |
|
|
1534
|
+
| **Query (Read)** | ~1-2ms | 2-4% increase | Keys hashed in KV storage |
|
|
1535
|
+
| **Insert (Write)** | ~2-3ms | 2-3% increase | Multi-key mappings protected |
|
|
1536
|
+
| **Update Mapping** | ~1-3ms | 1-2% increase | Existing keys remain secure |
|
|
1537
|
+
|
|
1538
|
+
#### Performance by Key Length
|
|
1539
|
+
|
|
1540
|
+
| Key Type | Example | Hash Time | Recommendation |
|
|
1541
|
+
| ------------------------ | ------------------------------ | ------------ | ----------------------- |
|
|
1542
|
+
| **Short keys** | `user-123` | ~0.5-1ms | Minimal impact |
|
|
1543
|
+
| **Medium keys** | `email:user@example.com` | ~1-2ms | Good balance |
|
|
1544
|
+
| **Long keys** | `session:very-long-token-here` | ~2-3ms | Consider key shortening |
|
|
1545
|
+
| **Multi-key operations** | 3+ lookup keys | ~3-5ms total | Benefits outweigh cost |
|
|
1546
|
+
|
|
1547
|
+
#### Hashing vs No-Hashing Trade-offs
|
|
1548
|
+
|
|
1549
|
+
```typescript
|
|
1550
|
+
// With hashing (default - recommended for production)
|
|
1551
|
+
const secureConfig = {
|
|
1552
|
+
hashShardMappings: true // Default
|
|
1553
|
+
// + Privacy: Sensitive data not visible in KV
|
|
1554
|
+
// + Security: Keys cannot be enumerated
|
|
1555
|
+
// - Performance: +1-3ms per operation
|
|
1556
|
+
// - Debugging: Original keys not recoverable
|
|
1557
|
+
};
|
|
1558
|
+
|
|
1559
|
+
// Without hashing (development/debugging only)
|
|
1560
|
+
const developmentConfig = {
|
|
1561
|
+
hashShardMappings: false
|
|
1562
|
+
// + Performance: No hashing overhead
|
|
1563
|
+
// + Debugging: Original keys visible in KV
|
|
1564
|
+
// - Privacy: Sensitive data exposed in KV keys
|
|
1565
|
+
// - Security: Keys can be enumerated
|
|
1566
|
+
};
|
|
1567
|
+
```
|
|
1568
|
+
|
|
1569
|
+
#### Optimization Recommendations
|
|
1570
|
+
|
|
1571
|
+
1. **Keep keys reasonably short** - Hash time scales with key length
|
|
1572
|
+
2. **Use hashing in production** - Security benefits outweigh minimal performance cost
|
|
1573
|
+
3. **Disable hashing for development** - When debugging shard distribution
|
|
1574
|
+
4. **Monitor hash performance** - Track operation latencies in high-volume scenarios
|
|
1575
|
+
|
|
1576
|
+
**Bottom Line**: SHA-256 hashing adds 1-3ms overhead but provides essential privacy and security benefits. The performance impact is minimal compared to network latency and D1 query time.
|
|
1577
|
+
|
|
1433
1578
|
### Real-World Scaling Benefits
|
|
1434
1579
|
|
|
1435
1580
|
#### Database Size Limits
|
|
@@ -1619,7 +1764,7 @@ const stats = await statsResponse.json();
|
|
|
1619
1764
|
"lastUpdated": 1672531200000
|
|
1620
1765
|
},
|
|
1621
1766
|
{
|
|
1622
|
-
"binding": "db-west",
|
|
1767
|
+
"binding": "db-west",
|
|
1623
1768
|
"count": 1458,
|
|
1624
1769
|
"lastUpdated": 1672531205000
|
|
1625
1770
|
}
|
|
@@ -1742,7 +1887,9 @@ async function monitorShardHealth(env: Env) {
|
|
|
1742
1887
|
}
|
|
1743
1888
|
```
|
|
1744
1889
|
|
|
1745
|
-
#### Error Handling
|
|
1890
|
+
#### Error Handling with ShardCoordinator
|
|
1891
|
+
|
|
1892
|
+
When using the ShardCoordinator, ensure you handle potential errors gracefully:
|
|
1746
1893
|
|
|
1747
1894
|
```typescript
|
|
1748
1895
|
try {
|
package/dist/index.js
CHANGED
|
@@ -1,19 +1,19 @@
|
|
|
1
|
-
var
|
|
1
|
+
var VJ=Object.defineProperty;var y=(J,Q)=>{for(var Z in Q)VJ(J,Z,{get:Q[Z],enumerable:!0,configurable:!0,set:($)=>Q[Z]=()=>$})};var c=(J,Q)=>()=>(J&&(Q=J(J=0)),Q);var L;var I=c(()=>{L=class L extends Error{code;constructor(J,Q){super(J);if(this.name="CollegeDBError",this.code=Q,Error.captureStackTrace)Error.captureStackTrace(this,L)}}});var g={};y(g,{KVShardMapper:()=>q});class q{kv;hashKeys;constructor(J,Q={}){this.kv=J,this.hashKeys=Q.hashShardMappings??!0}async hashKey(J){if(!this.hashKeys)return J;let Z=new TextEncoder().encode(J),$=await crypto.subtle.digest("SHA-256",Z),z=new Uint8Array($);return Array.from(z).map((V)=>V.toString(16).padStart(2,"0")).join("")}async getShardMapping(J){let Q=await this.hashKey(J),Z=`${w}${Q}`,$=await this.kv.get(Z,"json");if($)return $;let z=await this.kv.get(`${T}${Q}`,"json");if(z)return{shard:z.shard,createdAt:z.createdAt,updatedAt:z.updatedAt,originalKey:this.hashKeys?void 0:J};return null}async setShardMapping(J,Q,Z=[]){let $=[J,...Z],z=Date.now();if($.length===1){let W=await this.hashKey(J),V=`${w}${W}`,Y={shard:Q,createdAt:z,updatedAt:z,originalKey:this.hashKeys?void 0:J};await this.kv.put(V,JSON.stringify(Y))}else{let W=await this.hashKey(J),V=`${T}${W}`,Y={shard:Q,createdAt:z,updatedAt:z,keys:this.hashKeys?[]:$};await this.kv.put(V,JSON.stringify(Y));let O=$.map(async(j)=>{let U=await this.hashKey(j),x=`${w}${U}`,G={shard:Q,createdAt:z,updatedAt:z,originalKey:this.hashKeys?void 0:j};return this.kv.put(x,JSON.stringify(G))});await Promise.all(O)}}async updateShardMapping(J,Q){let Z=await this.getShardMapping(J);if(!Z)throw new L(`No existing mapping found for primary key: ${J}`,"MAPPING_NOT_FOUND");let $=await this.hashKey(J),z=`${w}${$}`,W=`${T}${$}`,V=await this.kv.get(W,"json");if(V){let Y={...V,shard:Q,updatedAt:Date.now()};await this.kv.put(W,JSON.stringify(Y));let O=V.keys.map(async(j)=>{let U=await this.hashKey(j),x=`${w}${U}`,G={...Z,shard:Q,updatedAt:Date.now()};return this.kv.put(x,JSON.stringify(G))});await Promise.all(O)}else{let Y={...Z,shard:Q,updatedAt:Date.now()};await this.kv.put(z,JSON.stringify(Y))}}async deleteShardMapping(J){let Q=await this.hashKey(J),Z=`${w}${Q}`,$=`${T}${Q}`,z=await this.kv.get($,"json");if(z){await this.kv.delete($);let W=z.keys.map(async(V)=>{let Y=await this.hashKey(V),O=`${w}${Y}`;return this.kv.delete(O)});await Promise.all(W)}else await this.kv.delete(Z)}async getKnownShards(){return await this.kv.get(d,"json")||[]}async setKnownShards(J){if(!J||J.length===0)return;await this.kv.put(d,JSON.stringify(J))}async addKnownShard(J){if(!J)return;let Q=await this.getKnownShards();if(!Q.includes(J))Q.push(J),await this.setKnownShards(Q)}async getKeysForShard(J){let Q=[],Z=await this.kv.list({prefix:w});for(let z of Z.keys){let W=await this.kv.get(z.name,"json");if(W?.shard===J){let V=z.name.replace(w,"");if(W.originalKey)Q.push(W.originalKey);else if(!this.hashKeys)Q.push(V)}}let $=await this.kv.list({prefix:T});for(let z of $.keys){let W=await this.kv.get(z.name,"json");if(W?.shard===J)Q.push(...W.keys)}return[...new Set(Q)]}async getShardKeyCounts(){let J={},Q=await this.kv.list({prefix:w});for(let $ of Q.keys){let z=await this.kv.get($.name,"json");if(z)J[z.shard]=(J[z.shard]||0)+1}let Z=await this.kv.list({prefix:T});for(let $ of Z.keys){let z=await this.kv.get($.name,"json");if(z)J[z.shard]=(J[z.shard]||0)+z.keys.length}return J}async clearAllMappings(){let Q=(await this.kv.list({prefix:w})).keys.map((z)=>this.kv.delete(z.name)),$=(await this.kv.list({prefix:T})).keys.map((z)=>this.kv.delete(z.name));await Promise.all([...Q,...$])}async addLookupKeys(J,Q){let Z=await this.getShardMapping(J);if(!Z)throw new L(`No existing mapping found for primary key: ${J}`,"MAPPING_NOT_FOUND");let $=await this.hashKey(J),z=`${T}${$}`,W=await this.kv.get(z,"json"),V=[J,...Q],Y=Date.now();if(!W)W={shard:Z.shard,createdAt:Z.createdAt,updatedAt:Y,keys:this.hashKeys?[]:V};else W={...W,updatedAt:Y,keys:this.hashKeys?[]:[...new Set([...W.keys,...V])]};await this.kv.put(z,JSON.stringify(W));let O=Q.map(async(j)=>{let U=await this.hashKey(j),x=`${w}${U}`,G={shard:Z.shard,createdAt:Z.createdAt,updatedAt:Y,originalKey:this.hashKeys?void 0:j};return this.kv.put(x,JSON.stringify(G))});await Promise.all(O)}async getAllLookupKeys(J){let Q=await this.hashKey(J),Z=`${T}${Q}`,$=await this.kv.get(Z,"json");if($)return $.keys;let z=await this.getShardMapping(J);if(z)return z.originalKey?[z.originalKey]:[J];throw new L(`No mapping found for key: ${J}`,"MAPPING_NOT_FOUND")}}var w="shard:",T="multikey:",d="known_shards";var k=c(()=>{I()});var S={};y(S,{validateTableForSharding:()=>f,schemaExists:()=>m,migrateRecord:()=>t,listTables:()=>_,integrateExistingDatabase:()=>s,dropSchema:()=>o,discoverExistingPrimaryKeys:()=>N,createSchemaAcrossShards:()=>n,createSchema:()=>h,createMappingsForExistingKeys:()=>i,clearShardMigrationCache:()=>JJ,clearMigrationCache:()=>e,checkMigrationNeeded:()=>a,autoDetectAndMigrate:()=>r});async function h(J,Q){let Z=Q.split(";").map(($)=>$.trim()).filter(($)=>$.length>0&&!$.startsWith("--"));for(let $ of Z)try{await J.prepare($).run()}catch(z){throw console.error("Failed to execute schema statement:",$,z),new L(`Schema migration failed: ${z}`,"SCHEMA_MIGRATION_FAILED")}}async function n(J,Q){let Z=Object.entries(J).map(([$,z])=>{return h(z,Q).catch((W)=>{throw new L(`Failed to create schema on shard ${$}: ${W.message}`,"SCHEMA_CREATION_FAILED")})});await Promise.all(Z)}async function m(J,Q){try{return await J.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='?'").bind(Q).first()!==null}catch{return!1}}async function o(J,...Q){for(let Z of Q)try{await J.prepare(`DROP TABLE IF EXISTS ${Z}`).run()}catch($){console.error(`Failed to drop table ${Z}:`,$)}}async function _(J){try{return(await J.prepare("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name").all()).results.map((Z)=>Z.name)}catch{return[]}}async function t(J,Q,Z,$){let z=await J.prepare(`SELECT * FROM ${$} WHERE id = ?`).bind(Z).first();if(!z)throw new L(`Record with primary key ${Z} not found in source database`,"RECORD_NOT_FOUND");if(!await m(Q,$))await h(Q,$);let W=Object.keys(z),V=W.map(()=>"?").join(", "),Y=W.map((j)=>z[j]),O=`INSERT OR REPLACE INTO ${$} (${W.join(", ")}) VALUES (${V})`;await Q.prepare(O).bind(...Y).run(),await J.prepare(`DELETE FROM ${$} WHERE id = ?`).bind(Z).run()}async function N(J,Q,Z="id"){try{return(await J.prepare(`SELECT ${Z} FROM ${Q}`).all()).results.map((z)=>String(z[Z]))}catch($){throw new L(`Failed to discover primary keys in table ${Q}: ${$}`,"DISCOVERY_FAILED")}}async function i(J,Q,Z,$){let z=Q.length;for(let W=0;W<J.length;W++){let V=J[W],Y;switch(Z){case"hash":let O=0;for(let U=0;U<V.length;U++){let x=V.charCodeAt(U);O=(O<<5)-O+x,O=O&O}let j=Math.abs(O)%z;Y=Q[j];break;case"random":Y=Q[Math.floor(Math.random()*z)];break;default:Y=Q[W%z];break}await $.setShardMapping(V,Y)}}async function f(J,Q,Z){let $=[],z=0;try{if(!await J.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").bind(Q).first())return $.push(`Table '${Q}' does not exist`),{isValid:!1,tableName:Q,primaryKeyColumn:Z,recordCount:0,issues:$};if(!(await J.prepare(`PRAGMA table_info(${Q})`).all()).results.some((j)=>j.name===Z&&j.pk===1))$.push(`Primary key column '${Z}' not found or not set as primary key`);if(z=(await J.prepare(`SELECT COUNT(*) as count FROM ${Q}`).first())?.count||0,z===0)$.push(`Table '${Q}' is empty`)}catch(W){$.push(`Database validation error: ${W}`)}return{isValid:$.length===0,tableName:Q,primaryKeyColumn:Z,recordCount:z,issues:$}}async function s(J,Q,Z,$={}){let{tables:z,primaryKeyColumn:W="id",strategy:V="hash",addShardMappingsTable:Y=!0,dryRun:O=!1}=$,j=[],U=0,x=0,G=0;try{let D=(z||await _(J)).filter((F)=>F!=="shard_mappings");for(let F of D)try{let R=await f(J,F,W);if(!R.isValid){j.push(`Table ${F}: ${R.issues.join(", ")}`);continue}let v=await N(J,F,W);if(v.length===0){j.push(`Table ${F} has no records to process`);continue}if(!O)for(let H of v)await Z.setShardMapping(H,Q),G++;U++,x+=v.length}catch(R){j.push(`Failed to process table ${F}: ${R}`)}if(Y&&!O){if(!(await _(J)).includes("shard_mappings"))await J.prepare(`
|
|
2
2
|
CREATE TABLE IF NOT EXISTS shard_mappings (
|
|
3
3
|
primary_key TEXT PRIMARY KEY,
|
|
4
4
|
shard_name TEXT NOT NULL,
|
|
5
5
|
created_at INTEGER NOT NULL,
|
|
6
6
|
updated_at INTEGER NOT NULL
|
|
7
|
-
);`.trim()).run()}if(!
|
|
8
|
-
SELECT ${
|
|
9
|
-
ORDER BY ${
|
|
10
|
-
LIMIT ?`.trim()).bind(
|
|
7
|
+
);`.trim()).run()}if(!O)await Z.addKnownShard(Q)}catch(X){j.push(`Integration failed: ${X}`)}return{success:j.length===0||j.length>0&&U>0,shardName:Q,tablesProcessed:U,totalRecords:x,mappingsCreated:G,issues:j}}async function r(J,Q,Z,$={}){let{primaryKeyColumn:z="id",tablesToCheck:W,skipCache:V=!1,maxRecordsToCheck:Y=1000}=$,O=`${Q}_migration_check`;if(!V&&B.has(O))return{migrationNeeded:!1,migrationPerformed:!1,recordsMigrated:0,tablesProcessed:0,issues:[]};let j=[],U=0,x=0,G=!1,X=!1;try{let{KVShardMapper:D}=await Promise.resolve().then(() => (k(),g)),F=new D(Z.kv,{hashShardMappings:Z.hashShardMappings}),R=await _(J),v=W||R.filter((H)=>H!=="shard_mappings"&&!H.startsWith("sqlite_")&&H!=="sqlite_sequence");if(v.length===0)return B.set(O,!0),{migrationNeeded:!1,migrationPerformed:!1,recordsMigrated:0,tablesProcessed:0,issues:[]};for(let H of v)try{let E=await f(J,H,z);if(!E.isValid||E.recordCount===0)continue;let $J=Math.min(Y,E.recordCount),zJ=await J.prepare(`
|
|
8
|
+
SELECT ${z} FROM ${H}
|
|
9
|
+
ORDER BY ${z}
|
|
10
|
+
LIMIT ?`.trim()).bind($J).all(),l=0,WJ=zJ.results.slice(0,10);for(let u of WJ){let P=String(u[z]);if(!await F.getShardMapping(P))l++,G=!0}if(l>0){console.log(`Auto-migrating table ${H} in shard ${Q} (${E.recordCount} records)`);let u=await N(J,H,z),P=0;for(let K of u)if(!await F.getShardMapping(K))await F.setShardMapping(K,Q),P++;U+=P,x++,X=!0,console.log(`Auto-migrated ${P} records from table ${H}`)}}catch(E){j.push(`Auto-migration failed for table ${H}: ${E}`)}if(X){if(await F.addKnownShard(Q),!R.includes("shard_mappings"))await J.prepare(`CREATE TABLE IF NOT EXISTS shard_mappings (
|
|
11
11
|
primary_key TEXT PRIMARY KEY,
|
|
12
12
|
shard_name TEXT NOT NULL,
|
|
13
13
|
created_at INTEGER NOT NULL,
|
|
14
14
|
updated_at INTEGER NOT NULL
|
|
15
15
|
);
|
|
16
|
-
`).run()}if(E.set(W,!0),F)console.log(`Auto-migration completed for shard ${Q}: ${L} records from ${X} tables`)}catch(T){O.push(`Auto-migration error: ${T}`)}return{migrationNeeded:x,migrationPerformed:F,recordsMigrated:L,tablesProcessed:X,issues:O}}async function r(J,Q,U){let Z=`${Q}_migration_check`;if(E.has(Z))return!1;try{let $=await P(J);if($.includes("shard_mappings"))return E.set(Z,!0),!1;let{KVShardMapper:Y}=await Promise.resolve().then(() => (v(),g)),V=new Y(U.kv),W=$.filter((O)=>O!=="shard_mappings"&&!O.startsWith("sqlite_")&&O!=="sqlite_sequence");for(let O of W.slice(0,3))try{if(((await J.prepare(`SELECT COUNT(*) as count FROM ${O} LIMIT 1`).first())?.count||0)>0){let x=await J.prepare(`SELECT id FROM ${O} LIMIT 1`).first();if(x){let F=String(x.id);if(!await V.getShardMapping(F))return!0}}}catch{continue}return!1}catch{return!1}}function a(){E.clear()}function e(J){let Q=`${J}_migration_check`;E.delete(Q)}var E;var B=b(()=>{D();E=new Map});D();v();var C=null;function WJ(J){if(C=J,J.shards&&Object.keys(J.shards).length>0&&!J.disableAutoMigration)QJ(J).catch((Q)=>{console.warn("Background auto-migration failed:",Q)})}async function JJ(J){if(C=J,J.shards&&Object.keys(J.shards).length>0&&!J.disableAutoMigration)try{await QJ(J)}catch(Q){console.warn("Auto migration failed:",Q)}}async function YJ(J,Q){return await JJ(J),await Q()}async function QJ(J){try{let{autoDetectAndMigrate:Q}=await Promise.resolve().then(() => (B(),k)),U=Object.keys(J.shards);console.log(`\uD83D\uDD0D Checking ${U.length} shards for existing data...`);let Z=U.map(async(Y)=>{let V=J.shards[Y];if(!V)return null;try{let W=await Q(V,Y,J,{maxRecordsToCheck:1000});return{shardName:Y,...W}}catch(W){return console.warn(`Auto-migration failed for shard ${Y}:`,W),null}}),z=(await Promise.all(Z)).filter((Y)=>Y?.migrationPerformed);if(z.length>0){let Y=z.reduce((V,W)=>V+(W?.recordsMigrated||0),0);console.log(`\uD83C\uDF89 Auto-migration completed! Migrated ${Y} records across ${z.length} shards`),z.forEach((V)=>{if(V)console.log(` ✅ ${V.shardName}: ${V.recordsMigrated} records from ${V.tablesProcessed} tables`)})}else console.log("✅ All shards ready - no migration needed")}catch(Q){console.warn("Background auto-migration setup failed:",Q)}}function LJ(){C=null}function H(){if(!C)throw new G("CollegeDB not initialized. Call initialize() first.","NOT_INITIALIZED");return C}function VJ(J){let Q=J.trim().toUpperCase();if(Q.startsWith("SELECT")||Q.startsWith("VALUES")||Q.startsWith("TABLE")||Q.startsWith("PRAGMA")||Q.startsWith("EXPLAIN")||Q.startsWith("WITH")||Q.startsWith("SHOW"))return"read";return"write"}function OJ(J,Q){let U=J.strategy||"hash";if(typeof U==="string")return U;return U[Q]}function GJ(J,Q){if(J===Q)return 0;let U={wnam:{lat:37.7749,lon:-122.4194},enam:{lat:40.7128,lon:-74.006},weur:{lat:51.5074,lon:-0.1278},eeur:{lat:52.52,lon:13.405},apac:{lat:35.6762,lon:139.6503},oc:{lat:-33.8688,lon:151.2093},me:{lat:25.2048,lon:55.2708},af:{lat:-26.2041,lon:28.0473}},Z=U[J],$=U[Q],z=Z.lat-$.lat,Y=Z.lon-$.lon;return Math.sqrt(z*z+Y*Y)}function XJ(J){let Q=J.cf;if(!Q||!Q.country)return"wnam";let{country:U,continent:Z}=Q;if(["US","CA","MX"].includes(U)){let $=Q.region||Q.regionCode||"",z=Q.timezone||"";if($.includes("CA")||$.includes("WA")||$.includes("OR")||$.includes("NV")||$.includes("AZ")||$.includes("UT")||z.includes("Pacific")||z.includes("America/Los_Angeles"))return"wnam";return"enam"}if(["GL","PM","BM"].includes(U))return"enam";if(["GB","IE","FR","ES","PT","NL","BE","LU","CH","AT","IT"].includes(U))return"weur";if(["DE","PL","CZ","SK","HU","SI","HR","BA","RS","ME","MK","AL","BG","RO","MD","UA","BY","LT","LV","EE","FI","SE","NO","DK","IS"].includes(U))return"eeur";if(U==="RU")return"eeur";if(["JP","KR","CN","HK","TW","MO","MN","KP"].includes(U))return"apac";if(["TH","VN","SG","MY","ID","PH","BN","KH","LA","MM","TL","IN","PK","BD","LK","NP","BT","MV","AF"].includes(U))return"apac";if(["AU","NZ","PG","FJ","NC","VU","SB","WS","TO","KI","NR","PW","FM","MH","TV"].includes(U))return"oc";if(["AE","SA","QA","KW","BH","OM","YE","IQ","IR","SY","LB","JO","IL","PS","TR","CY"].includes(U))return"me";if(Z==="AF"||["EG","LY","TN","DZ","MA","SD","SS","ET","ER","DJ","SO"].includes(U))return"af";if(["KZ","UZ","TM","TJ","KG"].includes(U))return"eeur";if(Z==="SA"||["BR","AR","CL","PE","CO","VE","EC","BO","PY","UY","GY","SR","GF"].includes(U))return"enam";if(["GT","BZ","SV","HN","NI","CR","PA","CU","JM","HT","DO","PR","TT","BB","GD","VC","LC","DM","AG","KN"].includes(U))return"enam";return"wnam"}function xJ(J,Q,U,Z){let $=Q.filter((L)=>U[L]);if($.length===0){let L=0;for(let x=0;x<Z.length;x++){let F=Z.charCodeAt(x);L=(L<<5)-L+F,L=L&L}let X=Math.abs(L)%Q.length;return Q[X]}let z=$.map((L)=>{let X=U[L],x=GJ(J,X.region),F=X.priority||1,T=x-F*0.1;return{shard:L,score:T,distance:x,priority:F}});z.sort((L,X)=>L.score-X.score);let Y=z[0].score,V=z.filter((L)=>Math.abs(L.score-Y)<0.01);if(V.length===1)return V[0].shard;let W=0;for(let L=0;L<Z.length;L++){let X=Z.charCodeAt(L);W=(W<<5)-W+X,W=W&W}let O=Math.abs(W)%V.length;return V[O].shard}async function FJ(J,Q="write"){let U=H(),Z=new A(U.kv),$=await Z.getShardMapping(J);if($)return $.shard;let z=Object.keys(U.shards);if(z.length===0)throw new G("No shards configured","NO_SHARDS");for(let W of z){let O=U.shards[W];if(!O)continue;try{let{autoDetectAndMigrate:L}=await Promise.resolve().then(() => (B(),k));if((await L(O,W,U,{maxRecordsToCheck:100})).migrationPerformed){let x=await Z.getShardMapping(J);if(x)return x.shard}}catch(L){console.warn(`Auto-migration check failed for shard ${W}:`,L)}}let Y,V=OJ(U,Q);if(U.coordinator)try{let W=U.coordinator.idFromName("default"),L=await U.coordinator.get(W).fetch("http://coordinator/allocate",{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({primaryKey:J,strategy:V,operationType:Q,targetRegion:U.targetRegion,shardLocations:U.shardLocations})});if(L.ok)Y=(await L.json()).shard;else Y=z[Math.floor(Math.random()*z.length)]}catch(W){console.warn("Coordinator allocation failed, falling back to local strategy:",W),Y=z[Math.floor(Math.random()*z.length)]}else switch(V){case"hash":let W=0;for(let L=0;L<J.length;L++){let X=J.charCodeAt(L);W=(W<<5)-W+X,W=W&W}let O=Math.abs(W)%z.length;Y=z[O]||z[0];break;case"location":if(!U.targetRegion){console.warn("Location strategy requires targetRegion in config, falling back to hash");let L=0;for(let x=0;x<J.length;x++){let F=J.charCodeAt(x);L=(L<<5)-L+F,L=L&L}let X=Math.abs(L)%z.length;Y=z[X]||z[0]}else Y=xJ(U.targetRegion,z,U.shardLocations||{},J);break;case"random":Y=z[Math.floor(Math.random()*z.length)]||z[0];break;default:Y=z[0];break}return await Z.setShardMapping(J,Y),Y}async function jJ(J,Q="write"){let U=H(),Z=await FJ(J,Q),$=U.shards[Z];if(!$)throw new G(`Shard ${Z} not found in configuration`,"SHARD_NOT_FOUND");return $}async function IJ(J,Q){let{createSchema:U}=await Promise.resolve().then(() => (B(),k));await U(J,Q)}async function K(J,Q){let U=VJ(Q);return(await jJ(J,U)).prepare(Q)}async function _J(J,Q,U=[]){let $=await(await K(J,Q)).bind(...U).run();if(!$.success)throw new G(`Query failed: ${$.error||"Unknown error"}`,"QUERY_FAILED");return $}async function HJ(J,Q,U=[]){let $=await(await K(J,Q)).bind(...U).all();if(!$.success)throw new G(`Query failed: ${$.error||"Unknown error"}`,"QUERY_FAILED");return $}async function TJ(J,Q,U=[]){return await(await K(J,Q)).bind(...U).first()}async function AJ(J,Q,U){let Z=H();if(!Z.shards[Q])throw new G(`Shard ${Q} not found in configuration`,"SHARD_NOT_FOUND");let $=new A(Z.kv),z=await $.getShardMapping(J);if(!z)throw new G(`No existing mapping found for primary key: ${J}`,"MAPPING_NOT_FOUND");if(z.shard!==Q){let{migrateRecord:Y}=await Promise.resolve().then(() => (B(),k)),V=Z.shards[z.shard],W=Z.shards[Q];if(!V||!W)throw new G("Source or target shard not available","SHARD_UNAVAILABLE");await Y(V,W,J,U)}await $.updateShardMapping(J,Q)}async function wJ(){let J=H();if(J.coordinator)try{let Q=J.coordinator.idFromName("default"),Z=await J.coordinator.get(Q).fetch("http://coordinator/shards");if(Z.ok)return await Z.json()}catch(Q){console.warn("Failed to get shards from coordinator:",Q)}return Object.keys(J.shards)}async function EJ(){let J=H();if(J.coordinator)try{let Z=J.coordinator.idFromName("default"),z=await J.coordinator.get(Z).fetch("http://coordinator/stats");if(z.ok)return await z.json()}catch(Z){console.warn("Failed to get stats from coordinator:",Z)}let U=await new A(J.kv).getShardKeyCounts();return Object.entries(J.shards).map(([Z,$])=>({binding:Z,count:U[Z]||0}))}async function MJ(J,Q,U=[]){let $=H().shards[J];if(!$)throw new G(`Shard ${J} not found`,"SHARD_NOT_FOUND");let z=await $.prepare(Q).bind(...U).run();if(!z.success)throw new G(`Query failed: ${z.error||"Unknown error"}`,"QUERY_FAILED");return z}async function RJ(J,Q,U=[]){let $=H().shards[J];if(!$)throw new G(`Shard ${J} not found`,"SHARD_NOT_FOUND");return await $.prepare(Q).bind(...U).all()}async function DJ(J,Q,U=[]){let $=H().shards[J];if(!$)throw new G(`Shard ${J} not found`,"SHARD_NOT_FOUND");return await $.prepare(Q).bind(...U).first()}async function PJ(){let J=H();if(await new A(J.kv).clearAllMappings(),J.coordinator)try{let U=J.coordinator.idFromName("default");await J.coordinator.get(U).fetch("http://coordinator/flush",{method:"POST"})}catch(U){console.warn("Failed to flush coordinator:",U)}}D();class y{state;constructor(J){this.state=J}async getState(){return await this.state.storage.get("coordinator_state")||{knownShards:[],shardStats:{},strategy:"round-robin",roundRobinIndex:0}}async saveState(J){await this.state.storage.put("coordinator_state",J)}async fetch(J){let U=new URL(J.url).pathname,Z=J.method;try{switch(`${Z} ${U}`){case"GET /shards":return this.handleListShards();case"POST /shards":return this.handleAddShard(J);case"DELETE /shards":return this.handleRemoveShard(J);case"GET /stats":return this.handleGetStats();case"POST /stats":return this.handleUpdateStats(J);case"POST /allocate":return this.handleAllocateShard(J);case"POST /flush":return this.handleFlush();case"GET /health":return new Response("OK",{status:200});default:return new Response("Not Found",{status:404})}}catch($){return console.error("ShardCoordinator error:",$),new Response("Internal Server Error",{status:500})}}async handleListShards(){let J=await this.getState();return new Response(JSON.stringify(J.knownShards),{headers:{"Content-Type":"application/json"}})}async handleAddShard(J){let{shard:Q}=await J.json();if(!Q||typeof Q!=="string")return new Response(JSON.stringify({error:"Missing or invalid shard parameter"}),{status:400,headers:{"Content-Type":"application/json"}});let U=await this.getState();if(!U.knownShards.includes(Q))U.knownShards.push(Q),U.shardStats[Q]={binding:Q,count:0,lastUpdated:Date.now()},await this.saveState(U);return new Response(JSON.stringify({success:!0}),{headers:{"Content-Type":"application/json"}})}async handleRemoveShard(J){let{shard:Q}=await J.json();if(!Q||typeof Q!=="string")return new Response(JSON.stringify({error:"Missing or invalid shard parameter"}),{status:400,headers:{"Content-Type":"application/json"}});let U=await this.getState(),Z=U.knownShards.indexOf(Q);if(Z>-1){if(U.knownShards.splice(Z,1),delete U.shardStats[Q],U.roundRobinIndex>=U.knownShards.length)U.roundRobinIndex=0;await this.saveState(U)}return new Response(JSON.stringify({success:!0}),{headers:{"Content-Type":"application/json"}})}async handleGetStats(){let J=await this.getState(),Q=Object.values(J.shardStats);return new Response(JSON.stringify(Q),{headers:{"Content-Type":"application/json"}})}async handleUpdateStats(J){let{shard:Q,count:U}=await J.json();if(!Q||typeof Q!=="string")return new Response(JSON.stringify({error:"Missing or invalid shard parameter"}),{status:400,headers:{"Content-Type":"application/json"}});if(U===void 0||typeof U!=="number")return new Response(JSON.stringify({error:"Missing or invalid count parameter"}),{status:400,headers:{"Content-Type":"application/json"}});let Z=await this.getState();if(Z.shardStats[Q])Z.shardStats[Q].count=U,Z.shardStats[Q].lastUpdated=Date.now(),await this.saveState(Z);return new Response(JSON.stringify({success:!0}),{headers:{"Content-Type":"application/json"}})}async handleAllocateShard(J){let{primaryKey:Q,strategy:U,operationType:Z}=await J.json();if(!Q||typeof Q!=="string")return new Response(JSON.stringify({error:"Missing or invalid primaryKey parameter"}),{status:400,headers:{"Content-Type":"application/json"}});let $=await this.getState();if($.knownShards.length===0)return new Response(JSON.stringify({error:"No shards available"}),{status:400,headers:{"Content-Type":"application/json"}});let z=this.resolveStrategy($.strategy,U,Z||"write"),Y=this.selectShard(Q,$,z);if(z==="round-robin")$.roundRobinIndex=($.roundRobinIndex+1)%$.knownShards.length,await this.saveState($);return new Response(JSON.stringify({shard:Y}),{headers:{"Content-Type":"application/json"}})}async handleFlush(){return await this.state.storage.deleteAll(),new Response(JSON.stringify({success:!0}),{headers:{"Content-Type":"application/json"}})}resolveStrategy(J,Q,U="write"){if(Q)return Q;if(typeof J==="string")return J;return J[U]}selectShard(J,Q,U){let Z=Q.knownShards;if(Z.length===0)throw new G("No shards available","NO_SHARDS");switch(U){case"round-robin":return Z[Q.roundRobinIndex]??Z[0];case"random":return Z[Math.floor(Math.random()*Z.length)];case"hash":let $=0;for(let W=0;W<J.length;W++){let O=J.charCodeAt(W);$=($<<5)-$+O,$=$&$}let z=Math.abs($)%Z.length;return Z[z];case"location":let Y=0;for(let W=0;W<J.length;W++){let O=J.charCodeAt(W);Y=(Y<<5)-Y+O,Y=Y&Y}let V=Math.abs(Y)%Z.length;return Z[V];default:return Z[0]}}async incrementShardCount(J){let Q=await this.getState();if(Q.shardStats[J])Q.shardStats[J].count++,Q.shardStats[J].lastUpdated=Date.now(),await this.saveState(Q)}async decrementShardCount(J){let Q=await this.getState();if(Q.shardStats[J]&&Q.shardStats[J].count>0)Q.shardStats[J].count--,Q.shardStats[J].lastUpdated=Date.now(),await this.saveState(Q)}}if(typeof global!=="undefined"){class J{data=new Map;async get(Z){return this.data.get(Z)}async put(Z,$){this.data.set(Z,$)}async delete(Z){return this.data.delete(Z)}async deleteAll(){this.data.clear()}async list(Z){if(!Z?.prefix)return new Map(this.data);let $=new Map;for(let[z,Y]of this.data.entries())if(z.startsWith(Z.prefix))$.set(z,Y);return $}}class Q{storage;constructor(){this.storage=new J}}class U{coordinator;mockState;constructor(){this.mockState=new Q,this.coordinator=new y(this.mockState)}async testShardAllocation(){await this.coordinator.fetch(new Request("http://test/shards",{method:"POST",body:JSON.stringify({shard:"db-east"})})),await this.coordinator.fetch(new Request("http://test/shards",{method:"POST",body:JSON.stringify({shard:"db-west"})}));let $=await(await this.coordinator.fetch(new Request("http://test/allocate",{method:"POST",body:JSON.stringify({primaryKey:"user-1",strategy:"round-robin"})}))).json(),Y=await(await this.coordinator.fetch(new Request("http://test/allocate",{method:"POST",body:JSON.stringify({primaryKey:"user-2",strategy:"round-robin"})}))).json();console.assert($.shard!==Y.shard,"Round-robin should alternate shards");let W=await(await this.coordinator.fetch(new Request("http://test/allocate",{method:"POST",body:JSON.stringify({primaryKey:"consistent-key",strategy:"hash"})}))).json(),L=await(await this.coordinator.fetch(new Request("http://test/allocate",{method:"POST",body:JSON.stringify({primaryKey:"consistent-key",strategy:"hash"})}))).json();console.assert(W.shard===L.shard,"Hash allocation should be consistent"),console.log("✅ Shard allocation tests passed")}async testShardStats(){await this.coordinator.fetch(new Request("http://test/flush",{method:"POST"})),await this.coordinator.fetch(new Request("http://test/shards",{method:"POST",body:JSON.stringify({shard:"db-stats-test"})})),await this.coordinator.fetch(new Request("http://test/stats",{method:"POST",body:JSON.stringify({shard:"db-stats-test",count:42})}));let $=await(await this.coordinator.fetch(new Request("http://test/stats",{method:"GET"}))).json();console.assert($.length===1,"Should have one shard stat"),console.assert($[0]?.binding==="db-stats-test","Should have correct binding name"),console.assert($[0]?.count===42,"Should have correct count"),console.log("✅ Shard stats tests passed")}async testErrorHandling(){await this.coordinator.fetch(new Request("http://test/flush",{method:"POST"}));let Z=await this.coordinator.fetch(new Request("http://test/allocate",{method:"POST",body:JSON.stringify({primaryKey:"test-key"})}));console.assert(Z.status===400,"Should return 400 for no shards available");let $=await this.coordinator.fetch(new Request("http://test/invalid",{method:"GET"}));console.assert($.status===404,"Should return 404 for invalid endpoint"),console.log("✅ Error handling tests passed")}async testCountManagement(){await this.coordinator.fetch(new Request("http://test/shards",{method:"POST",body:JSON.stringify({shard:"db-count-test"})})),await this.coordinator.incrementShardCount("db-count-test"),await this.coordinator.incrementShardCount("db-count-test");let Z=await this.coordinator.fetch(new Request("http://test/stats",{method:"GET"})),$=await Z.json(),z=$.find((V)=>V.binding==="db-count-test");console.assert(z?.count===2,"Count should be 2 after two increments"),await this.coordinator.decrementShardCount("db-count-test"),Z=await this.coordinator.fetch(new Request("http://test/stats",{method:"GET"})),$=await Z.json();let Y=$.find((V)=>V.binding==="db-count-test");console.assert(Y?.count===1,"Count should be 1 after decrement"),console.log("✅ Count management tests passed")}async runAllTests(){console.log("\uD83E\uDDEA Running ShardCoordinator tests...");try{return await this.testShardAllocation(),await this.testShardStats(),await this.testErrorHandling(),await this.testCountManagement(),console.log("\uD83C\uDF89 All ShardCoordinator tests passed!"),!0}catch(Z){return console.error("❌ ShardCoordinator tests failed:",Z),!1}}}globalThis.testShardCoordinator=()=>new U}D();v();B();export{N as validateTableForSharding,c as schemaExists,MJ as runShard,_J as run,LJ as resetConfig,AJ as reassignShard,K as prepare,o as migrateRecord,P as listTables,wJ as listKnownShards,t as integrateExistingDatabase,JJ as initializeAsync,WJ as initialize,EJ as getShardStats,XJ as getClosestRegionFromIP,PJ as flush,DJ as firstShard,TJ as first,n as dropSchema,S as discoverExistingPrimaryKeys,d as createSchemaAcrossShards,IJ as createSchema,i as createMappingsForExistingKeys,YJ as collegedb,e as clearShardMigrationCache,a as clearMigrationCache,r as checkMigrationNeeded,s as autoDetectAndMigrate,RJ as allShard,HJ as all,y as ShardCoordinator,A as KVShardMapper,G as CollegeDBError};
|
|
16
|
+
`).run()}if(B.set(O,!0),X)console.log(`Auto-migration completed for shard ${Q}: ${U} records from ${x} tables`)}catch(D){j.push(`Auto-migration error: ${D}`)}return{migrationNeeded:G,migrationPerformed:X,recordsMigrated:U,tablesProcessed:x,issues:j}}async function a(J,Q,Z){let $=`${Q}_migration_check`;if(B.has($))return!1;try{let z=await _(J);if(z.includes("shard_mappings"))return B.set($,!0),!1;let{KVShardMapper:V}=await Promise.resolve().then(() => (k(),g)),Y=new V(Z.kv,{hashShardMappings:Z.hashShardMappings}),O=z.filter((j)=>j!=="shard_mappings"&&!j.startsWith("sqlite_")&&j!=="sqlite_sequence");for(let j of O.slice(0,3))try{if(((await J.prepare(`SELECT COUNT(*) as count FROM ${j} LIMIT 1`).first())?.count||0)>0){let G=await J.prepare(`SELECT id FROM ${j} LIMIT 1`).first();if(G){let X=String(G.id);if(!await Y.getShardMapping(X))return!0}}}catch{continue}return!1}catch{return!1}}function e(){B.clear()}function JJ(J){let Q=`${J}_migration_check`;B.delete(Q)}var B;var C=c(()=>{I();B=new Map});I();k();var M=null;function OJ(J){if(M=J,J.shards&&Object.keys(J.shards).length>0&&!J.disableAutoMigration)ZJ(J).catch((Q)=>{console.warn("Background auto-migration failed:",Q)})}async function QJ(J){if(M=J,J.shards&&Object.keys(J.shards).length>0&&!J.disableAutoMigration)try{await ZJ(J)}catch(Q){console.warn("Auto migration failed:",Q)}}async function UJ(J,Q){return await QJ(J),await Q()}async function ZJ(J){try{let{autoDetectAndMigrate:Q}=await Promise.resolve().then(() => (C(),S)),Z=Object.keys(J.shards);console.log(`\uD83D\uDD0D Checking ${Z.length} shards for existing data...`);let $=Z.map(async(V)=>{let Y=J.shards[V];if(!Y)return null;try{let O=await Q(Y,V,J,{maxRecordsToCheck:1000});return{shardName:V,...O}}catch(O){return console.warn(`Auto-migration failed for shard ${V}:`,O),null}}),W=(await Promise.all($)).filter((V)=>V?.migrationPerformed);if(W.length>0){let V=W.reduce((Y,O)=>Y+(O?.recordsMigrated||0),0);console.log(`\uD83C\uDF89 Auto-migration completed! Migrated ${V} records across ${W.length} shards`),W.forEach((Y)=>{if(Y)console.log(` ✅ ${Y.shardName}: ${Y.recordsMigrated} records from ${Y.tablesProcessed} tables`)})}else console.log("✅ All shards ready - no migration needed")}catch(Q){console.warn("Background auto-migration setup failed:",Q)}}function YJ(){M=null}function A(){if(!M)throw new L("CollegeDB not initialized. Call initialize() first.","NOT_INITIALIZED");return M}function jJ(J){let Q=J.trim().toUpperCase();if(Q.startsWith("SELECT")||Q.startsWith("VALUES")||Q.startsWith("TABLE")||Q.startsWith("PRAGMA")||Q.startsWith("EXPLAIN")||Q.startsWith("WITH")||Q.startsWith("SHOW"))return"read";return"write"}function xJ(J,Q){let Z=J.strategy||"hash";if(typeof Z==="string")return Z;return Z[Q]}function GJ(J,Q){if(J===Q)return 0;let Z={wnam:{lat:37.7749,lon:-122.4194},enam:{lat:40.7128,lon:-74.006},weur:{lat:51.5074,lon:-0.1278},eeur:{lat:52.52,lon:13.405},apac:{lat:35.6762,lon:139.6503},oc:{lat:-33.8688,lon:151.2093},me:{lat:25.2048,lon:55.2708},af:{lat:-26.2041,lon:28.0473}},$=Z[J],z=Z[Q],W=$.lat-z.lat,V=$.lon-z.lon;return Math.sqrt(W*W+V*V)}function LJ(J){let Q=J.cf;if(!Q||!Q.country)return"wnam";let{country:Z,continent:$}=Q;if(["US","CA","MX"].includes(Z)){let z=Q.region||Q.regionCode||"",W=Q.timezone||"";if(z.includes("CA")||z.includes("WA")||z.includes("OR")||z.includes("NV")||z.includes("AZ")||z.includes("UT")||W.includes("Pacific")||W.includes("America/Los_Angeles"))return"wnam";return"enam"}if(["GL","PM","BM"].includes(Z))return"enam";if(["GB","IE","FR","ES","PT","NL","BE","LU","CH","AT","IT"].includes(Z))return"weur";if(["DE","PL","CZ","SK","HU","SI","HR","BA","RS","ME","MK","AL","BG","RO","MD","UA","BY","LT","LV","EE","FI","SE","NO","DK","IS"].includes(Z))return"eeur";if(Z==="RU")return"eeur";if(["JP","KR","CN","HK","TW","MO","MN","KP"].includes(Z))return"apac";if(["TH","VN","SG","MY","ID","PH","BN","KH","LA","MM","TL","IN","PK","BD","LK","NP","BT","MV","AF"].includes(Z))return"apac";if(["AU","NZ","PG","FJ","NC","VU","SB","WS","TO","KI","NR","PW","FM","MH","TV"].includes(Z))return"oc";if(["AE","SA","QA","KW","BH","OM","YE","IQ","IR","SY","LB","JO","IL","PS","TR","CY"].includes(Z))return"me";if($==="AF"||["EG","LY","TN","DZ","MA","SD","SS","ET","ER","DJ","SO"].includes(Z))return"af";if(["KZ","UZ","TM","TJ","KG"].includes(Z))return"eeur";if($==="SA"||["BR","AR","CL","PE","CO","VE","EC","BO","PY","UY","GY","SR","GF"].includes(Z))return"enam";if(["GT","BZ","SV","HN","NI","CR","PA","CU","JM","HT","DO","PR","TT","BB","GD","VC","LC","DM","AG","KN"].includes(Z))return"enam";return"wnam"}function XJ(J,Q,Z,$){let z=Q.filter((U)=>Z[U]);if(z.length===0){let U=0;for(let G=0;G<$.length;G++){let X=$.charCodeAt(G);U=(U<<5)-U+X,U=U&U}let x=Math.abs(U)%Q.length;return Q[x]}let W=z.map((U)=>{let x=Z[U],G=GJ(J,x.region),X=x.priority||1,D=G-X*0.1;return{shard:U,score:D,distance:G,priority:X}});W.sort((U,x)=>U.score-x.score);let V=W[0].score,Y=W.filter((U)=>Math.abs(U.score-V)<0.01);if(Y.length===1)return Y[0].shard;let O=0;for(let U=0;U<$.length;U++){let x=$.charCodeAt(U);O=(O<<5)-O+x,O=O&O}let j=Math.abs(O)%Y.length;return Y[j].shard}async function FJ(J,Q="write"){let Z=A(),$=new q(Z.kv,{hashShardMappings:Z.hashShardMappings}),z=await $.getShardMapping(J);if(z)return z.shard;let W=Object.keys(Z.shards);if(W.length===0)throw new L("No shards configured","NO_SHARDS");for(let O of W){let j=Z.shards[O];if(!j)continue;try{let{autoDetectAndMigrate:U}=await Promise.resolve().then(() => (C(),S));if((await U(j,O,Z,{maxRecordsToCheck:100})).migrationPerformed){let G=await $.getShardMapping(J);if(G)return G.shard}}catch(U){console.warn(`Auto-migration check failed for shard ${O}:`,U)}}let V,Y=xJ(Z,Q);if(Z.coordinator)try{let O=Z.coordinator.idFromName("default"),U=await Z.coordinator.get(O).fetch("http://coordinator/allocate",{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({primaryKey:J,strategy:Y,operationType:Q,targetRegion:Z.targetRegion,shardLocations:Z.shardLocations})});if(U.ok)V=(await U.json()).shard;else V=W[Math.floor(Math.random()*W.length)]}catch(O){console.warn("Coordinator allocation failed, falling back to local strategy:",O),V=W[Math.floor(Math.random()*W.length)]}else switch(Y){case"hash":let O=0;for(let U=0;U<J.length;U++){let x=J.charCodeAt(U);O=(O<<5)-O+x,O=O&O}let j=Math.abs(O)%W.length;V=W[j]||W[0];break;case"location":if(!Z.targetRegion){console.warn("Location strategy requires targetRegion in config, falling back to hash");let U=0;for(let G=0;G<J.length;G++){let X=J.charCodeAt(G);U=(U<<5)-U+X,U=U&U}let x=Math.abs(U)%W.length;V=W[x]||W[0]}else V=XJ(Z.targetRegion,W,Z.shardLocations||{},J);break;case"random":V=W[Math.floor(Math.random()*W.length)]||W[0];break;default:V=W[0];break}return await $.setShardMapping(J,V),V}async function HJ(J,Q="write"){let Z=A(),$=await FJ(J,Q),z=Z.shards[$];if(!z)throw new L(`Shard ${$} not found in configuration`,"SHARD_NOT_FOUND");return z}async function wJ(J,Q){let{createSchema:Z}=await Promise.resolve().then(() => (C(),S));await Z(J,Q)}async function b(J,Q){let Z=jJ(Q);return(await HJ(J,Z)).prepare(Q)}async function TJ(J,Q,Z=[]){let z=await(await b(J,Q)).bind(...Z).run();if(!z.success)throw new L(`Query failed: ${z.error||"Unknown error"}`,"QUERY_FAILED");return z}async function AJ(J,Q,Z=[]){let z=await(await b(J,Q)).bind(...Z).all();if(!z.success)throw new L(`Query failed: ${z.error||"Unknown error"}`,"QUERY_FAILED");return z}async function DJ(J,Q,Z=[]){return await(await b(J,Q)).bind(...Z).first()}async function qJ(J,Q,Z){let $=A();if(!$.shards[Q])throw new L(`Shard ${Q} not found in configuration`,"SHARD_NOT_FOUND");let z=new q($.kv,{hashShardMappings:$.hashShardMappings}),W=await z.getShardMapping(J);if(!W)throw new L(`No existing mapping found for primary key: ${J}`,"MAPPING_NOT_FOUND");if(W.shard!==Q){let{migrateRecord:V}=await Promise.resolve().then(() => (C(),S)),Y=$.shards[W.shard],O=$.shards[Q];if(!Y||!O)throw new L("Source or target shard not available","SHARD_UNAVAILABLE");await V(Y,O,J,Z)}await z.updateShardMapping(J,Q)}async function RJ(){let J=A();if(J.coordinator)try{let Q=J.coordinator.idFromName("default"),$=await J.coordinator.get(Q).fetch("http://coordinator/shards");if($.ok)return await $.json()}catch(Q){console.warn("Failed to get shards from coordinator:",Q)}return Object.keys(J.shards)}async function BJ(){let J=A();if(J.coordinator)try{let $=J.coordinator.idFromName("default"),W=await J.coordinator.get($).fetch("http://coordinator/stats");if(W.ok)return await W.json()}catch($){console.warn("Failed to get stats from coordinator:",$)}let Z=await new q(J.kv,{hashShardMappings:J.hashShardMappings}).getShardKeyCounts();return Object.entries(J.shards).map(([$,z])=>({binding:$,count:Z[$]||0}))}async function vJ(J,Q,Z=[]){let z=A().shards[J];if(!z)throw new L(`Shard ${J} not found`,"SHARD_NOT_FOUND");let W=await z.prepare(Q).bind(...Z).run();if(!W.success)throw new L(`Query failed: ${W.error||"Unknown error"}`,"QUERY_FAILED");return W}async function EJ(J,Q,Z=[]){let z=A().shards[J];if(!z)throw new L(`Shard ${J} not found`,"SHARD_NOT_FOUND");return await z.prepare(Q).bind(...Z).all()}async function IJ(J,Q,Z=[]){let z=A().shards[J];if(!z)throw new L(`Shard ${J} not found`,"SHARD_NOT_FOUND");return await z.prepare(Q).bind(...Z).first()}async function _J(){let J=A();if(await new q(J.kv,{hashShardMappings:J.hashShardMappings}).clearAllMappings(),J.coordinator)try{let Z=J.coordinator.idFromName("default");await J.coordinator.get(Z).fetch("http://coordinator/flush",{method:"POST"})}catch(Z){console.warn("Failed to flush coordinator:",Z)}}I();class p{state;constructor(J){this.state=J}async getState(){return await this.state.storage.get("coordinator_state")||{knownShards:[],shardStats:{},strategy:"round-robin",roundRobinIndex:0}}async saveState(J){await this.state.storage.put("coordinator_state",J)}async fetch(J){let Z=new URL(J.url).pathname,$=J.method;try{switch(`${$} ${Z}`){case"GET /shards":return this.handleListShards();case"POST /shards":return this.handleAddShard(J);case"DELETE /shards":return this.handleRemoveShard(J);case"GET /stats":return this.handleGetStats();case"POST /stats":return this.handleUpdateStats(J);case"POST /allocate":return this.handleAllocateShard(J);case"POST /flush":return this.handleFlush();case"GET /health":return new Response("OK",{status:200});default:return new Response("Not Found",{status:404})}}catch(z){return console.error("ShardCoordinator error:",z),new Response("Internal Server Error",{status:500})}}async handleListShards(){let J=await this.getState();return new Response(JSON.stringify(J.knownShards),{headers:{"Content-Type":"application/json"}})}async handleAddShard(J){let{shard:Q}=await J.json();if(!Q||typeof Q!=="string")return new Response(JSON.stringify({error:"Missing or invalid shard parameter"}),{status:400,headers:{"Content-Type":"application/json"}});let Z=await this.getState();if(!Z.knownShards.includes(Q))Z.knownShards.push(Q),Z.shardStats[Q]={binding:Q,count:0,lastUpdated:Date.now()},await this.saveState(Z);return new Response(JSON.stringify({success:!0}),{headers:{"Content-Type":"application/json"}})}async handleRemoveShard(J){let{shard:Q}=await J.json();if(!Q||typeof Q!=="string")return new Response(JSON.stringify({error:"Missing or invalid shard parameter"}),{status:400,headers:{"Content-Type":"application/json"}});let Z=await this.getState(),$=Z.knownShards.indexOf(Q);if($>-1){if(Z.knownShards.splice($,1),delete Z.shardStats[Q],Z.roundRobinIndex>=Z.knownShards.length)Z.roundRobinIndex=0;await this.saveState(Z)}return new Response(JSON.stringify({success:!0}),{headers:{"Content-Type":"application/json"}})}async handleGetStats(){let J=await this.getState(),Q=Object.values(J.shardStats);return new Response(JSON.stringify(Q),{headers:{"Content-Type":"application/json"}})}async handleUpdateStats(J){let{shard:Q,count:Z}=await J.json();if(!Q||typeof Q!=="string")return new Response(JSON.stringify({error:"Missing or invalid shard parameter"}),{status:400,headers:{"Content-Type":"application/json"}});if(Z===void 0||typeof Z!=="number")return new Response(JSON.stringify({error:"Missing or invalid count parameter"}),{status:400,headers:{"Content-Type":"application/json"}});let $=await this.getState();if($.shardStats[Q])$.shardStats[Q].count=Z,$.shardStats[Q].lastUpdated=Date.now(),await this.saveState($);return new Response(JSON.stringify({success:!0}),{headers:{"Content-Type":"application/json"}})}async handleAllocateShard(J){let{primaryKey:Q,strategy:Z,operationType:$}=await J.json();if(!Q||typeof Q!=="string")return new Response(JSON.stringify({error:"Missing or invalid primaryKey parameter"}),{status:400,headers:{"Content-Type":"application/json"}});let z=await this.getState();if(z.knownShards.length===0)return new Response(JSON.stringify({error:"No shards available"}),{status:400,headers:{"Content-Type":"application/json"}});let W=this.resolveStrategy(z.strategy,Z,$||"write"),V=this.selectShard(Q,z,W);if(W==="round-robin")z.roundRobinIndex=(z.roundRobinIndex+1)%z.knownShards.length,await this.saveState(z);return new Response(JSON.stringify({shard:V}),{headers:{"Content-Type":"application/json"}})}async handleFlush(){return await this.state.storage.deleteAll(),new Response(JSON.stringify({success:!0}),{headers:{"Content-Type":"application/json"}})}resolveStrategy(J,Q,Z="write"){if(Q)return Q;if(typeof J==="string")return J;return J[Z]}selectShard(J,Q,Z){let $=Q.knownShards;if($.length===0)throw new L("No shards available","NO_SHARDS");switch(Z){case"round-robin":return $[Q.roundRobinIndex]??$[0];case"random":return $[Math.floor(Math.random()*$.length)];case"hash":let z=0;for(let O=0;O<J.length;O++){let j=J.charCodeAt(O);z=(z<<5)-z+j,z=z&z}let W=Math.abs(z)%$.length;return $[W];case"location":let V=0;for(let O=0;O<J.length;O++){let j=J.charCodeAt(O);V=(V<<5)-V+j,V=V&V}let Y=Math.abs(V)%$.length;return $[Y];default:return $[0]}}async incrementShardCount(J){let Q=await this.getState();if(Q.shardStats[J])Q.shardStats[J].count++,Q.shardStats[J].lastUpdated=Date.now(),await this.saveState(Q)}async decrementShardCount(J){let Q=await this.getState();if(Q.shardStats[J]&&Q.shardStats[J].count>0)Q.shardStats[J].count--,Q.shardStats[J].lastUpdated=Date.now(),await this.saveState(Q)}}if(typeof global!=="undefined"){class J{data=new Map;async get($){return this.data.get($)}async put($,z){this.data.set($,z)}async delete($){return this.data.delete($)}async deleteAll(){this.data.clear()}async list($){if(!$?.prefix)return new Map(this.data);let z=new Map;for(let[W,V]of this.data.entries())if(W.startsWith($.prefix))z.set(W,V);return z}}class Q{storage;constructor(){this.storage=new J}}class Z{coordinator;mockState;constructor(){this.mockState=new Q,this.coordinator=new p(this.mockState)}async testShardAllocation(){await this.coordinator.fetch(new Request("http://test/shards",{method:"POST",body:JSON.stringify({shard:"db-east"})})),await this.coordinator.fetch(new Request("http://test/shards",{method:"POST",body:JSON.stringify({shard:"db-west"})}));let z=await(await this.coordinator.fetch(new Request("http://test/allocate",{method:"POST",body:JSON.stringify({primaryKey:"user-1",strategy:"round-robin"})}))).json(),V=await(await this.coordinator.fetch(new Request("http://test/allocate",{method:"POST",body:JSON.stringify({primaryKey:"user-2",strategy:"round-robin"})}))).json();console.assert(z.shard!==V.shard,"Round-robin should alternate shards");let O=await(await this.coordinator.fetch(new Request("http://test/allocate",{method:"POST",body:JSON.stringify({primaryKey:"consistent-key",strategy:"hash"})}))).json(),U=await(await this.coordinator.fetch(new Request("http://test/allocate",{method:"POST",body:JSON.stringify({primaryKey:"consistent-key",strategy:"hash"})}))).json();console.assert(O.shard===U.shard,"Hash allocation should be consistent"),console.log("✅ Shard allocation tests passed")}async testShardStats(){await this.coordinator.fetch(new Request("http://test/flush",{method:"POST"})),await this.coordinator.fetch(new Request("http://test/shards",{method:"POST",body:JSON.stringify({shard:"db-stats-test"})})),await this.coordinator.fetch(new Request("http://test/stats",{method:"POST",body:JSON.stringify({shard:"db-stats-test",count:42})}));let z=await(await this.coordinator.fetch(new Request("http://test/stats",{method:"GET"}))).json();console.assert(z.length===1,"Should have one shard stat"),console.assert(z[0]?.binding==="db-stats-test","Should have correct binding name"),console.assert(z[0]?.count===42,"Should have correct count"),console.log("✅ Shard stats tests passed")}async testErrorHandling(){await this.coordinator.fetch(new Request("http://test/flush",{method:"POST"}));let $=await this.coordinator.fetch(new Request("http://test/allocate",{method:"POST",body:JSON.stringify({primaryKey:"test-key"})}));console.assert($.status===400,"Should return 400 for no shards available");let z=await this.coordinator.fetch(new Request("http://test/invalid",{method:"GET"}));console.assert(z.status===404,"Should return 404 for invalid endpoint"),console.log("✅ Error handling tests passed")}async testCountManagement(){await this.coordinator.fetch(new Request("http://test/shards",{method:"POST",body:JSON.stringify({shard:"db-count-test"})})),await this.coordinator.incrementShardCount("db-count-test"),await this.coordinator.incrementShardCount("db-count-test");let $=await this.coordinator.fetch(new Request("http://test/stats",{method:"GET"})),z=await $.json(),W=z.find((Y)=>Y.binding==="db-count-test");console.assert(W?.count===2,"Count should be 2 after two increments"),await this.coordinator.decrementShardCount("db-count-test"),$=await this.coordinator.fetch(new Request("http://test/stats",{method:"GET"})),z=await $.json();let V=z.find((Y)=>Y.binding==="db-count-test");console.assert(V?.count===1,"Count should be 1 after decrement"),console.log("✅ Count management tests passed")}async runAllTests(){console.log("\uD83E\uDDEA Running ShardCoordinator tests...");try{return await this.testShardAllocation(),await this.testShardStats(),await this.testErrorHandling(),await this.testCountManagement(),console.log("\uD83C\uDF89 All ShardCoordinator tests passed!"),!0}catch($){return console.error("❌ ShardCoordinator tests failed:",$),!1}}}globalThis.testShardCoordinator=()=>new Z}I();k();C();export{f as validateTableForSharding,m as schemaExists,vJ as runShard,TJ as run,YJ as resetConfig,qJ as reassignShard,b as prepare,t as migrateRecord,_ as listTables,RJ as listKnownShards,s as integrateExistingDatabase,QJ as initializeAsync,OJ as initialize,BJ as getShardStats,LJ as getClosestRegionFromIP,_J as flush,IJ as firstShard,DJ as first,o as dropSchema,N as discoverExistingPrimaryKeys,n as createSchemaAcrossShards,wJ as createSchema,i as createMappingsForExistingKeys,UJ as collegedb,JJ as clearShardMigrationCache,e as clearMigrationCache,a as checkMigrationNeeded,r as autoDetectAndMigrate,EJ as allShard,AJ as all,p as ShardCoordinator,q as KVShardMapper,L as CollegeDBError};
|
|
17
17
|
|
|
18
|
-
//# debugId=
|
|
18
|
+
//# debugId=85FADF7DE7B481B664756E2164756E21
|
|
19
19
|
//# sourceMappingURL=index.js.map
|