@earth-app/collegedb 1.0.6 → 1.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +108 -21
- package/dist/durable.d.ts +1 -0
- package/dist/durable.d.ts.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -5
- package/dist/index.js.map +7 -7
- package/dist/kvmap.d.ts +6 -2
- package/dist/kvmap.d.ts.map +1 -1
- package/dist/migrations.d.ts.map +1 -1
- package/dist/router.d.ts +15 -0
- package/dist/router.d.ts.map +1 -1
- package/dist/types.d.ts +8 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# CollegeDB
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
_Cloudflare D1 Horizontal Sharding Router_
|
|
4
4
|
|
|
5
5
|
[](https://www.typescriptlang.org/)
|
|
6
6
|
[](https://github.com/earth-app/CollegeDB/issues)
|
|
@@ -65,15 +65,15 @@ CollegeDB provides a sharding layer on top of Cloudflare D1 databases, enabling
|
|
|
65
65
|
## Installation
|
|
66
66
|
|
|
67
67
|
```bash
|
|
68
|
-
bun add collegedb
|
|
68
|
+
bun add @earth-app/collegedb
|
|
69
69
|
# or
|
|
70
|
-
npm install collegedb
|
|
70
|
+
npm install @earth-app/collegedb
|
|
71
71
|
```
|
|
72
72
|
|
|
73
73
|
## Basic Usage
|
|
74
74
|
|
|
75
75
|
```typescript
|
|
76
|
-
import { collegedb, createSchema, run, first } from 'collegedb';
|
|
76
|
+
import { collegedb, createSchema, run, first } from '@earth-app/collegedb';
|
|
77
77
|
|
|
78
78
|
// Initialize with your Cloudflare bindings (existing databases work automatically!)
|
|
79
79
|
collegedb(
|
|
@@ -104,7 +104,7 @@ collegedb(
|
|
|
104
104
|
### Geographic Distribution Example
|
|
105
105
|
|
|
106
106
|
```typescript
|
|
107
|
-
import { collegedb, first, run } from 'collegedb';
|
|
107
|
+
import { collegedb, first, run } from '@earth-app/collegedb';
|
|
108
108
|
|
|
109
109
|
// Optimize for North American users with geographic sharding
|
|
110
110
|
collegedb(
|
|
@@ -141,7 +141,7 @@ collegedb(
|
|
|
141
141
|
### Mixed Strategy Example
|
|
142
142
|
|
|
143
143
|
```typescript
|
|
144
|
-
import { collegedb, first, run, type MixedShardingStrategy } from 'collegedb';
|
|
144
|
+
import { collegedb, first, run, type MixedShardingStrategy } from '@earth-app/collegedb';
|
|
145
145
|
|
|
146
146
|
// Use location strategy for writes (optimal data placement) and hash for reads (optimal performance)
|
|
147
147
|
const mixedStrategy: MixedShardingStrategy = {
|
|
@@ -194,7 +194,7 @@ This approach provides:
|
|
|
194
194
|
CollegeDB supports **multiple lookup keys** for the same record, allowing you to query by username, email, ID, or any unique identifier. Keys are automatically hashed with SHA-256 for security and privacy.
|
|
195
195
|
|
|
196
196
|
```typescript
|
|
197
|
-
import { collegedb, first, run, KVShardMapper } from 'collegedb';
|
|
197
|
+
import { collegedb, first, run, KVShardMapper } from '@earth-app/collegedb';
|
|
198
198
|
|
|
199
199
|
collegedb(
|
|
200
200
|
{
|
|
@@ -291,7 +291,7 @@ CollegeDB supports **seamless, automatic integration** with existing D1 database
|
|
|
291
291
|
4. **KV Namespace**: A Cloudflare KV namespace for storing shard mappings
|
|
292
292
|
|
|
293
293
|
```typescript
|
|
294
|
-
import { collegedb, first, run } from 'collegedb';
|
|
294
|
+
import { collegedb, first, run } from '@earth-app/collegedb';
|
|
295
295
|
|
|
296
296
|
// Add your existing databases as shards - that's it!
|
|
297
297
|
collegedb(
|
|
@@ -321,7 +321,7 @@ collegedb(
|
|
|
321
321
|
You can manually validate databases before integration if needed:
|
|
322
322
|
|
|
323
323
|
```typescript
|
|
324
|
-
import { validateTableForSharding, listTables } from 'collegedb';
|
|
324
|
+
import { validateTableForSharding, listTables } from '@earth-app/collegedb';
|
|
325
325
|
|
|
326
326
|
// Check database structure
|
|
327
327
|
const tables = await listTables(env.ExistingDB);
|
|
@@ -343,7 +343,7 @@ for (const table of tables) {
|
|
|
343
343
|
If you want to inspect existing data before automatic migration:
|
|
344
344
|
|
|
345
345
|
```typescript
|
|
346
|
-
import { discoverExistingPrimaryKeys } from 'collegedb';
|
|
346
|
+
import { discoverExistingPrimaryKeys } from '@earth-app/collegedb';
|
|
347
347
|
|
|
348
348
|
// Discover all user IDs in existing users table
|
|
349
349
|
const userIds = await discoverExistingPrimaryKeys(env.ExistingDB, 'users');
|
|
@@ -358,7 +358,7 @@ const orderIds = await discoverExistingPrimaryKeys(env.ExistingDB, 'orders', 'or
|
|
|
358
358
|
For complete control over the integration process:
|
|
359
359
|
|
|
360
360
|
```typescript
|
|
361
|
-
import { integrateExistingDatabase, KVShardMapper } from 'collegedb';
|
|
361
|
+
import { integrateExistingDatabase, KVShardMapper } from '@earth-app/collegedb';
|
|
362
362
|
|
|
363
363
|
const mapper = new KVShardMapper(env.KV);
|
|
364
364
|
|
|
@@ -386,7 +386,7 @@ if (result.success) {
|
|
|
386
386
|
After integration, initialize CollegeDB with your existing databases as shards:
|
|
387
387
|
|
|
388
388
|
```typescript
|
|
389
|
-
import { initialize, first } from 'collegedb';
|
|
389
|
+
import { initialize, first } from '@earth-app/collegedb';
|
|
390
390
|
|
|
391
391
|
// Include existing databases as shards
|
|
392
392
|
initialize({
|
|
@@ -409,7 +409,7 @@ const user = await first('existing-user-123', 'SELECT * FROM users WHERE id = ?'
|
|
|
409
409
|
The simplest possible integration - just add your existing databases:
|
|
410
410
|
|
|
411
411
|
```typescript
|
|
412
|
-
import { initialize, first, run } from 'collegedb';
|
|
412
|
+
import { initialize, first, run } from '@earth-app/collegedb';
|
|
413
413
|
|
|
414
414
|
export default {
|
|
415
415
|
async fetch(request: Request, env: Env): Promise<Response> {
|
|
@@ -479,12 +479,12 @@ console.log(`Would process ${testResult.totalRecords} records from ${testResult.
|
|
|
479
479
|
|
|
480
480
|
```typescript
|
|
481
481
|
// Simple rollback - clear all mappings
|
|
482
|
-
import { KVShardMapper } from 'collegedb';
|
|
482
|
+
import { KVShardMapper } from '@earth-app/collegedb';
|
|
483
483
|
const mapper = new KVShardMapper(env.KV);
|
|
484
484
|
await mapper.clearAllMappings(); // Returns to pre-migration state
|
|
485
485
|
|
|
486
486
|
// Or clear cache to force re-detection
|
|
487
|
-
import { clearMigrationCache } from 'collegedb';
|
|
487
|
+
import { clearMigrationCache } from '@earth-app/collegedb';
|
|
488
488
|
clearMigrationCache(); // Forces fresh migration check
|
|
489
489
|
```
|
|
490
490
|
|
|
@@ -549,6 +549,7 @@ for (const [table, pkColumn] of Object.entries(customIntegration)) {
|
|
|
549
549
|
| `reassignShard(key, newShard)` | Move primary key to different shard | `string, string` |
|
|
550
550
|
| `listKnownShards()` | Get list of available shards | `void` |
|
|
551
551
|
| `getShardStats()` | Get statistics for all shards | `void` |
|
|
552
|
+
| `getDatabaseSizeForShard(shard)` | Get size of a specific shard in bytes | `string` |
|
|
552
553
|
| `flush()` | Clear all shard mappings (development only) | `void` |
|
|
553
554
|
|
|
554
555
|
### Drop-in Replacement Functions
|
|
@@ -612,7 +613,7 @@ The `ShardCoordinator` is an optional Durable Object that provides centralized s
|
|
|
612
613
|
#### Usage Example
|
|
613
614
|
|
|
614
615
|
```typescript
|
|
615
|
-
import { ShardCoordinator } from 'collegedb';
|
|
616
|
+
import { ShardCoordinator } from '@earth-app/collegedb';
|
|
616
617
|
|
|
617
618
|
// Export for Cloudflare Workers runtime
|
|
618
619
|
export { ShardCoordinator };
|
|
@@ -652,6 +653,7 @@ interface CollegeDBConfig {
|
|
|
652
653
|
shardLocations?: Record<string, ShardLocation>;
|
|
653
654
|
disableAutoMigration?: boolean; // Default: false
|
|
654
655
|
hashShardMappings?: boolean; // Default: true
|
|
656
|
+
maxDatabaseSize?: number; // Default: undefined (no limit)
|
|
655
657
|
}
|
|
656
658
|
```
|
|
657
659
|
|
|
@@ -702,6 +704,91 @@ const mixedStrategyConfig: CollegeDBConfig = {
|
|
|
702
704
|
};
|
|
703
705
|
```
|
|
704
706
|
|
|
707
|
+
### Database Size Management
|
|
708
|
+
|
|
709
|
+
CollegeDB supports automatic size-based shard exclusion to prevent individual shards from becoming too large. This feature helps maintain optimal performance and prevents hitting D1 storage limits.
|
|
710
|
+
|
|
711
|
+
#### Configuration
|
|
712
|
+
|
|
713
|
+
```typescript
|
|
714
|
+
const config: CollegeDBConfig = {
|
|
715
|
+
kv: env.KV,
|
|
716
|
+
shards: {
|
|
717
|
+
'db-east': env.DB_EAST,
|
|
718
|
+
'db-west': env.DB_WEST,
|
|
719
|
+
'db-central': env.DB_CENTRAL
|
|
720
|
+
},
|
|
721
|
+
strategy: 'hash',
|
|
722
|
+
maxDatabaseSize: 500 * 1024 * 1024 // 500 MB limit per shard
|
|
723
|
+
};
|
|
724
|
+
```
|
|
725
|
+
|
|
726
|
+
#### How It Works
|
|
727
|
+
|
|
728
|
+
When `maxDatabaseSize` is configured:
|
|
729
|
+
|
|
730
|
+
1. **Allocation Phase**: Before allocating new records, CollegeDB checks each shard's size using efficient SQLite pragmas
|
|
731
|
+
2. **Size Filtering**: Shards exceeding the limit are excluded from new allocations
|
|
732
|
+
3. **Fallback Protection**: If all shards exceed the limit, allocation continues to prevent complete failure
|
|
733
|
+
4. **Existing Records**: Records already mapped to oversized shards remain accessible
|
|
734
|
+
|
|
735
|
+
#### Size Check Implementation
|
|
736
|
+
|
|
737
|
+
The size check uses SQLite's `PRAGMA page_count` and `PRAGMA page_size` for accurate, low-overhead size calculation:
|
|
738
|
+
|
|
739
|
+
```sql
|
|
740
|
+
-- Efficient size calculation (used internally)
|
|
741
|
+
PRAGMA page_count; -- Returns number of database pages
|
|
742
|
+
PRAGMA page_size; -- Returns size of each page in bytes
|
|
743
|
+
-- Total size = page_count × page_size
|
|
744
|
+
```
|
|
745
|
+
|
|
746
|
+
#### Usage Examples
|
|
747
|
+
|
|
748
|
+
```typescript
|
|
749
|
+
// Conservative limit for high-performance scenarios
|
|
750
|
+
const performanceConfig: CollegeDBConfig = {
|
|
751
|
+
// ... other config
|
|
752
|
+
maxDatabaseSize: 100 * 1024 * 1024, // 100 MB per shard
|
|
753
|
+
strategy: 'round-robin' // Ensures even distribution
|
|
754
|
+
};
|
|
755
|
+
|
|
756
|
+
// Standard production limit
|
|
757
|
+
const productionConfig: CollegeDBConfig = {
|
|
758
|
+
// ... other config
|
|
759
|
+
maxDatabaseSize: 1024 * 1024 * 1024, // 1 GB per shard
|
|
760
|
+
strategy: 'hash' // Consistent allocation
|
|
761
|
+
};
|
|
762
|
+
|
|
763
|
+
// Check individual shard sizes
|
|
764
|
+
import { getDatabaseSizeForShard } from '@earth-app/collegedb';
|
|
765
|
+
|
|
766
|
+
const eastSize = await getDatabaseSizeForShard('db-east');
|
|
767
|
+
console.log(`East shard: ${Math.round(eastSize / 1024 / 1024)} MB`);
|
|
768
|
+
```
|
|
769
|
+
|
|
770
|
+
#### Debug Logging
|
|
771
|
+
|
|
772
|
+
Enable debug logging to monitor size-based exclusions:
|
|
773
|
+
|
|
774
|
+
```typescript
|
|
775
|
+
const config: CollegeDBConfig = {
|
|
776
|
+
// ... other config
|
|
777
|
+
maxDatabaseSize: 500 * 1024 * 1024,
|
|
778
|
+
debug: true // Logs when shards are excluded due to size
|
|
779
|
+
};
|
|
780
|
+
|
|
781
|
+
// Console output example:
|
|
782
|
+
// "Excluded 2 shards due to size limits: db-east, db-central"
|
|
783
|
+
```
|
|
784
|
+
|
|
785
|
+
#### Performance Impact
|
|
786
|
+
|
|
787
|
+
- **Size Check Frequency**: Only performed during new allocations (not on reads)
|
|
788
|
+
- **Query Efficiency**: Uses fast SQLite pragmas (microsecond execution time)
|
|
789
|
+
- **Parallel Execution**: Size checks run concurrently across all shards
|
|
790
|
+
- **Caching**: No caching implemented to ensure accurate real-time limits
|
|
791
|
+
|
|
705
792
|
### Types
|
|
706
793
|
|
|
707
794
|
CollegeDB exports TypeScript types for better development experience and type safety:
|
|
@@ -719,7 +806,7 @@ CollegeDB exports TypeScript types for better development experience and type sa
|
|
|
719
806
|
#### Mixed Strategy Configuration
|
|
720
807
|
|
|
721
808
|
```typescript
|
|
722
|
-
import type { MixedShardingStrategy, CollegeDBConfig } from 'collegedb';
|
|
809
|
+
import type { MixedShardingStrategy, CollegeDBConfig } from '@earth-app/collegedb';
|
|
723
810
|
|
|
724
811
|
// Type-safe mixed strategy configuration
|
|
725
812
|
const mixedStrategy: MixedShardingStrategy = {
|
|
@@ -914,7 +1001,7 @@ Create your main worker file with ShardCoordinator export:
|
|
|
914
1001
|
|
|
915
1002
|
```typescript
|
|
916
1003
|
// src/index.ts
|
|
917
|
-
import { collegedb, ShardCoordinator, first, run } from 'collegedb';
|
|
1004
|
+
import { collegedb, ShardCoordinator, first, run } from '@earth-app/collegedb';
|
|
918
1005
|
|
|
919
1006
|
// IMPORTANT: Export ShardCoordinator for Cloudflare Workers runtime
|
|
920
1007
|
export { ShardCoordinator };
|
|
@@ -978,7 +1065,7 @@ wrangler deploy --env production
|
|
|
978
1065
|
#### Using CollegeDB Functions
|
|
979
1066
|
|
|
980
1067
|
```typescript
|
|
981
|
-
import { getShardStats, listKnownShards } from 'collegedb';
|
|
1068
|
+
import { getShardStats, listKnownShards } from '@earth-app/collegedb';
|
|
982
1069
|
|
|
983
1070
|
// Get detailed statistics
|
|
984
1071
|
const stats = await getShardStats();
|
|
@@ -1086,7 +1173,7 @@ export default {
|
|
|
1086
1173
|
### Shard Rebalancing
|
|
1087
1174
|
|
|
1088
1175
|
```typescript
|
|
1089
|
-
import { reassignShard } from 'collegedb';
|
|
1176
|
+
import { reassignShard } from '@earth-app/collegedb';
|
|
1090
1177
|
|
|
1091
1178
|
// Move a primary key to a different shard
|
|
1092
1179
|
await reassignShard('user-123', 'db-west');
|
|
@@ -1693,7 +1780,7 @@ class_name = "ShardCoordinator"
|
|
|
1693
1780
|
#### Basic Usage with ShardCoordinator
|
|
1694
1781
|
|
|
1695
1782
|
```typescript
|
|
1696
|
-
import { collegedb, ShardCoordinator } from 'collegedb';
|
|
1783
|
+
import { collegedb, ShardCoordinator } from '@earth-app/collegedb';
|
|
1697
1784
|
|
|
1698
1785
|
// Export the Durable Object class for Cloudflare Workers
|
|
1699
1786
|
export { ShardCoordinator };
|
package/dist/durable.d.ts
CHANGED
|
@@ -212,6 +212,7 @@ export declare class ShardCoordinator {
|
|
|
212
212
|
* @param primaryKey - The primary key to allocate a shard for
|
|
213
213
|
* @param state - Current coordinator state containing available shards
|
|
214
214
|
* @param strategy - The allocation strategy to use
|
|
215
|
+
* @param eligibleShards - Optional filtered list of shards to choose from
|
|
215
216
|
* @returns The selected shard binding name
|
|
216
217
|
* @throws {CollegeDBError} If no shards are available
|
|
217
218
|
* @example
|
package/dist/durable.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"durable.d.ts","sourceRoot":"","sources":["../src/durable.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAIpE;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,qBAAa,gBAAgB;IAC5B;;;OAGG;IACH,OAAO,CAAC,KAAK,CAAqB;IAElC;;;OAGG;gBACS,KAAK,EAAE,kBAAkB;IAIrC;;;;;;;;;;OAUG;YACW,QAAQ;IAYtB;;;;;;;;;;OAUG;YACW,SAAS;IAIvB;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACG,KAAK,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC;IAgChD;;;;;;OAMG;YACW,gBAAgB;IAO9B;;;;;;;;;OASG;YACW,cAAc;IA4B5B;;;;;;;;;OASG;YACW,iBAAiB;IA6B/B;;;;;;OAMG;YACW,cAAc;IAQ5B;;;;;;;;OAQG;YACW,iBAAiB;IA+B/B;;;;;;;;;;;;;;;OAeG;YACW,mBAAmB;
|
|
1
|
+
{"version":3,"file":"durable.d.ts","sourceRoot":"","sources":["../src/durable.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAIpE;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,qBAAa,gBAAgB;IAC5B;;;OAGG;IACH,OAAO,CAAC,KAAK,CAAqB;IAElC;;;OAGG;gBACS,KAAK,EAAE,kBAAkB;IAIrC;;;;;;;;;;OAUG;YACW,QAAQ;IAYtB;;;;;;;;;;OAUG;YACW,SAAS;IAIvB;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACG,KAAK,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC;IAgChD;;;;;;OAMG;YACW,gBAAgB;IAO9B;;;;;;;;;OASG;YACW,cAAc;IA4B5B;;;;;;;;;OASG;YACW,iBAAiB;IA6B/B;;;;;;OAMG;YACW,cAAc;IAQ5B;;;;;;;;OAQG;YACW,iBAAiB;IA+B/B;;;;;;;;;;;;;;;OAeG;YACW,mBAAmB;IAsCjC;;;;;;;;;;OAUG;YACW,WAAW;IAOzB;;;;;;;;OAQG;IACH,OAAO,CAAC,eAAe;IAmBvB;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,OAAO,CAAC,WAAW;IAgGnB;;;;;;;;;;;OAWG;IACG,mBAAmB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IASvD;;;;;;;;;;;OAWG;IACG,mBAAmB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAQvD"}
|
package/dist/index.d.ts
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* @author Gregory Mitchell
|
|
9
9
|
* @license MIT
|
|
10
10
|
*/
|
|
11
|
-
export { all, allAllShards, allShard, collegedb, createSchema, first, firstAllShards, firstShard, flush, getClosestRegionFromIP, getShardStats, initialize, initializeAsync, listKnownShards, prepare, reassignShard, resetConfig, run, runAllShards, runShard } from './router.js';
|
|
11
|
+
export { all, allAllShards, allShard, collegedb, createSchema, first, firstAllShards, firstShard, flush, getClosestRegionFromIP, getDatabaseSizeForShard, getShardStats, initialize, initializeAsync, listKnownShards, prepare, reassignShard, resetConfig, run, runAllShards, runShard } from './router.js';
|
|
12
12
|
export { ShardCoordinator } from './durable.js';
|
|
13
13
|
export { CollegeDBError } from './errors.js';
|
|
14
14
|
export { KVShardMapper } from './kvmap.js';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,EACN,GAAG,EACH,YAAY,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,KAAK,EACL,cAAc,EACd,UAAU,EACV,KAAK,EACL,sBAAsB,EACtB,aAAa,EACb,UAAU,EACV,eAAe,EACf,eAAe,EACf,OAAO,EACP,aAAa,EACb,WAAW,EACX,GAAG,EACH,YAAY,EACZ,QAAQ,EACR,MAAM,aAAa,CAAC;AAGrB,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAG3C,OAAO,EACN,oBAAoB,EACpB,oBAAoB,EACpB,mBAAmB,EACnB,wBAAwB,EACxB,6BAA6B,EAC7B,wBAAwB,EACxB,2BAA2B,EAC3B,kCAAkC,EAClC,UAAU,EACV,yBAAyB,EACzB,UAAU,EACV,aAAa,EACb,YAAY,EACZ,wBAAwB,EACxB,KAAK,kBAAkB,EACvB,KAAK,iBAAiB,EACtB,KAAK,gBAAgB,EACrB,MAAM,iBAAiB,CAAC;AAGzB,YAAY,EACX,eAAe,EACf,QAAQ,EACR,GAAG,EACH,qBAAqB,EACrB,aAAa,EACb,qBAAqB,EACrB,aAAa,EACb,YAAY,EACZ,UAAU,EACV,gBAAgB,EAChB,MAAM,YAAY,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,EACN,GAAG,EACH,YAAY,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,KAAK,EACL,cAAc,EACd,UAAU,EACV,KAAK,EACL,sBAAsB,EACtB,uBAAuB,EACvB,aAAa,EACb,UAAU,EACV,eAAe,EACf,eAAe,EACf,OAAO,EACP,aAAa,EACb,WAAW,EACX,GAAG,EACH,YAAY,EACZ,QAAQ,EACR,MAAM,aAAa,CAAC;AAGrB,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAG3C,OAAO,EACN,oBAAoB,EACpB,oBAAoB,EACpB,mBAAmB,EACnB,wBAAwB,EACxB,6BAA6B,EAC7B,wBAAwB,EACxB,2BAA2B,EAC3B,kCAAkC,EAClC,UAAU,EACV,yBAAyB,EACzB,UAAU,EACV,aAAa,EACb,YAAY,EACZ,wBAAwB,EACxB,KAAK,kBAAkB,EACvB,KAAK,iBAAiB,EACtB,KAAK,gBAAgB,EACrB,MAAM,iBAAiB,CAAC;AAGzB,YAAY,EACX,eAAe,EACf,QAAQ,EACR,GAAG,EACH,qBAAqB,EACrB,aAAa,EACb,qBAAqB,EACrB,aAAa,EACb,YAAY,EACZ,UAAU,EACV,gBAAgB,EAChB,MAAM,YAAY,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,19 +1,19 @@
|
|
|
1
|
-
var
|
|
1
|
+
var GJ=Object.defineProperty;var a=(J,Z)=>{for(var $ in Z)GJ(J,$,{get:Z[$],enumerable:!0,configurable:!0,set:(W)=>Z[$]=()=>W})};var d=(J,Z)=>()=>(J&&(Z=J(J=0)),Z);var G;var k=d(()=>{G=class G extends Error{code;constructor(J,Z){super(J);if(this.name="CollegeDBError",this.code=Z,Error.captureStackTrace)Error.captureStackTrace(this,G)}}});var n={};a(n,{KVShardMapper:()=>I});class I{kv;hashKeys;hashCache=new Map;constructor(J,Z={}){this.kv=J,this.hashKeys=Z.hashShardMappings??!0}async hashKey(J){if(!this.hashKeys)return J;let Z=this.hashCache.get(J);if(Z)return Z;let W=new TextEncoder().encode(J),Q=await crypto.subtle.digest("SHA-256",W),V=new Uint8Array(Q),O=Array.from(V).map((U)=>U.toString(16).padStart(2,"0")).join("");if(this.hashCache.size<1e4)this.hashCache.set(J,O);return O}async getShardMapping(J){let Z=await this.hashKey(J),$=`${R}${Z}`,W=await this.kv.get($,"json");if(W)return W;let Q=await this.kv.get(`${v}${Z}`,"json");if(Q)return{shard:Q.shard,createdAt:Q.createdAt,updatedAt:Q.updatedAt,originalKey:this.hashKeys?void 0:J};return null}async setShardMapping(J,Z,$=[]){let W=[J,...$],Q=Date.now();if(W.length===1){let V=await this.hashKey(J),O=`${R}${V}`,U={shard:Z,createdAt:Q,updatedAt:Q,originalKey:this.hashKeys?void 0:J};await this.kv.put(O,JSON.stringify(U))}else{let V=await this.hashKey(J),O=`${v}${V}`,U=this.hashKeys?await Promise.all(W.map((j)=>this.hashKey(j))):W,Y={shard:Z,createdAt:Q,updatedAt:Q,keys:U};await this.kv.put(O,JSON.stringify(Y));let H=W.map(async(j)=>{let X=await this.hashKey(j),x=`${R}${X}`,T={shard:Z,createdAt:Q,updatedAt:Q,originalKey:this.hashKeys?void 0:j};return this.kv.put(x,JSON.stringify(T))});await Promise.all(H)}}async updateShardMapping(J,Z){let $=await this.getShardMapping(J);if(!$)throw new G(`No existing mapping found for primary key: ${J}`,"MAPPING_NOT_FOUND");let W=await this.hashKey(J),Q=`${R}${W}`,V=`${v}${W}`,O=await this.kv.get(V,"json");if(O){let U=Date.now(),Y={...O,shard:Z,updatedAt:U};await this.kv.put(V,JSON.stringify(Y));let j=(O.keys.length>0?this.hashKeys?O.keys:O.keys:[await this.hashKey(J)]).map(async(X)=>{let x=`${R}${X}`,T={...$,shard:Z,updatedAt:U};return this.kv.put(x,JSON.stringify(T))});await Promise.all(j)}else{let U={...$,shard:Z,updatedAt:Date.now()};await this.kv.put(Q,JSON.stringify(U))}}async deleteShardMapping(J){let Z=await this.hashKey(J),$=`${R}${Z}`,W=`${v}${Z}`,Q=await this.kv.get(W,"json");if(Q){await this.kv.delete(W);let O=(Q.keys.length>0?this.hashKeys?Q.keys:Q.keys:[await this.hashKey(J)]).map(async(U)=>{let Y=`${R}${U}`;return this.kv.delete(Y)});await Promise.all(O)}else await this.kv.delete($)}async getKnownShards(){return await this.kv.get(e,"json")||[]}async setKnownShards(J){if(!J||J.length===0)return;await this.kv.put(e,JSON.stringify(J))}async addKnownShard(J){if(!J)return;let Z=await this.getKnownShards();if(!Z.includes(J))Z.push(J),await this.setKnownShards(Z)}async getKeysForShard(J){let Z=[],$=await this.kv.list({prefix:R});for(let Q of $.keys){let V=await this.kv.get(Q.name,"json");if(V?.shard===J){let O=Q.name.replace(R,"");if(V.originalKey)Z.push(V.originalKey);else if(!this.hashKeys)Z.push(O)}}let W=await this.kv.list({prefix:v});for(let Q of W.keys){let V=await this.kv.get(Q.name,"json");if(V?.shard===J)Z.push(...V.keys)}return[...new Set(Z)]}async getShardKeyCounts(){let J={},Z=await this.kv.list({prefix:R});for(let W of Z.keys){let Q=await this.kv.get(W.name,"json");if(Q)J[Q.shard]=(J[Q.shard]||0)+1}let $=await this.kv.list({prefix:v});for(let W of $.keys){let Q=await this.kv.get(W.name,"json");if(Q)J[Q.shard]=(J[Q.shard]||0)+Q.keys.length}return J}async clearAllMappings(){let Z=(await this.kv.list({prefix:R})).keys.map((Q)=>this.kv.delete(Q.name)),W=(await this.kv.list({prefix:v})).keys.map((Q)=>this.kv.delete(Q.name));await Promise.all([...Z,...W])}async addLookupKeys(J,Z){let $=await this.getShardMapping(J);if(!$)throw new G(`No existing mapping found for primary key: ${J}`,"MAPPING_NOT_FOUND");let W=await this.hashKey(J),Q=`${v}${W}`,V=await this.kv.get(Q,"json"),O=[J,...Z],U=Date.now();if(!V){let H=this.hashKeys?await Promise.all(O.map((j)=>this.hashKey(j))):O;V={shard:$.shard,createdAt:$.createdAt,updatedAt:U,keys:H}}else{let H=this.hashKeys?await Promise.all(O.map((j)=>this.hashKey(j))):O;V={...V,updatedAt:U,keys:[...new Set([...V.keys,...H])]}}await this.kv.put(Q,JSON.stringify(V));let Y=Z.map(async(H)=>{let j=await this.hashKey(H),X=`${R}${j}`,x={shard:$.shard,createdAt:$.createdAt,updatedAt:U,originalKey:this.hashKeys?void 0:H};return this.kv.put(X,JSON.stringify(x))});await Promise.all(Y)}async getAllLookupKeys(J){let Z=await this.hashKey(J),$=`${v}${Z}`,W=await this.kv.get($,"json");if(W)return W.keys;let Q=await this.getShardMapping(J);if(Q)return Q.originalKey?[Q.originalKey]:[J];throw new G(`No mapping found for key: ${J}`,"MAPPING_NOT_FOUND")}}var R="shard:",v="multikey:",e="known_shards";var S=d(()=>{k()});var m={};a(m,{validateTableForSharding:()=>p,schemaExists:()=>i,migrateRecord:()=>$J,listTables:()=>N,integrateExistingDatabase:()=>WJ,dropSchema:()=>ZJ,discoverExistingRecordsWithColumns:()=>g,discoverExistingPrimaryKeys:()=>h,createSchemaAcrossShards:()=>JJ,createSchema:()=>t,createMappingsForExistingKeys:()=>QJ,clearShardMigrationCache:()=>YJ,clearMigrationCache:()=>UJ,checkMigrationNeeded:()=>OJ,autoDetectAndMigrate:()=>VJ});async function t(J,Z){let $=Z.split(";").map((W)=>W.trim()).filter((W)=>W.length>0&&!W.startsWith("--"));for(let W of $)try{await J.prepare(W).run()}catch(Q){throw console.error("Failed to execute schema statement:",W,Q),new G(`Schema migration failed: ${Q}`,"SCHEMA_MIGRATION_FAILED")}}async function JJ(J,Z){let $=Object.entries(J).map(([W,Q])=>{return t(Q,Z).catch((V)=>{throw new G(`Failed to create schema on shard ${W}: ${V.message}`,"SCHEMA_CREATION_FAILED")})});await Promise.all($)}async function i(J,Z){try{return await J.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").bind(Z).first()!==null}catch{return!1}}async function ZJ(J,...Z){for(let $ of Z)try{await J.prepare(`DROP TABLE IF EXISTS ${$}`).run()}catch(W){console.error(`Failed to drop table ${$}:`,W)}}async function N(J){try{return(await J.prepare("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name").all()).results.map(($)=>$.name)}catch{return[]}}async function $J(J,Z,$,W){let Q=await J.prepare(`SELECT * FROM ${W} WHERE id = ?`).bind($).first();if(!Q)throw new G(`Record with primary key ${$} not found in source database`,"RECORD_NOT_FOUND");if(!await i(Z,W))await t(Z,W);let V=Object.keys(Q),O=V.map(()=>"?").join(", "),U=V.map((H)=>Q[H]),Y=`INSERT OR REPLACE INTO ${W} (${V.join(", ")}) VALUES (${O})`;await Z.prepare(Y).bind(...U).run(),await J.prepare(`DELETE FROM ${W} WHERE id = ?`).bind($).run()}async function h(J,Z,$="id"){try{return(await J.prepare(`SELECT ${$} FROM ${Z}`).all()).results.map((Q)=>String(Q[$]))}catch(W){throw new G(`Failed to discover primary keys in table ${Z}: ${W}`,"DISCOVERY_FAILED")}}async function g(J,Z,$="id"){try{let W=`${Z}_columns`,Q;if(o.has(W))Q=o.get(W).map((Y)=>Y.name);else{let H=(await J.prepare(`PRAGMA table_info(${Z})`).all()).results.map((j)=>({name:j.name,type:j.type}));o.set(W,H),Q=H.map((j)=>j.name)}let V=[$];if(Q.includes("username"))V.push("username");if(Q.includes("email"))V.push("email");if(Q.includes("name"))V.push("name");let O=`SELECT ${V.join(", ")} FROM ${Z}`;return(await J.prepare(O).all()).results}catch(W){throw new G(`Failed to discover records with columns in table ${Z}: ${W}`,"DISCOVERY_FAILED")}}async function QJ(J,Z,$,W){let Q=Z.length;for(let V=0;V<J.length;V++){let O=J[V],U;switch($){case"hash":let Y=0;for(let j=0;j<O.length;j++){let X=O.charCodeAt(j);Y=(Y<<5)-Y+X,Y=Y&Y}let H=Math.abs(Y)%Q;U=Z[H];break;case"random":U=Z[Math.floor(Math.random()*Q)];break;default:U=Z[V%Q];break}await W.setShardMapping(O,U)}}async function p(J,Z,$){let W=[],Q=0;try{if(!await J.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name=?").bind(Z).first())return W.push(`Table '${Z}' does not exist`),{isValid:!1,tableName:Z,primaryKeyColumn:$,recordCount:0,issues:W};if(!(await J.prepare(`PRAGMA table_info(${Z})`).all()).results.some((H)=>H.name===$&&H.pk===1))W.push(`Primary key column '${$}' not found or not set as primary key`);if(Q=(await J.prepare(`SELECT COUNT(*) as count FROM ${Z}`).first())?.count||0,Q===0)W.push(`Table '${Z}' is empty`)}catch(V){W.push(`Database validation error: ${V}`)}return{isValid:W.length===0,tableName:Z,primaryKeyColumn:$,recordCount:Q,issues:W}}async function WJ(J,Z,$,W={}){let{tables:Q,primaryKeyColumn:V="id",strategy:O="hash",addShardMappingsTable:U=!0,dryRun:Y=!1,migrateOtherColumns:H=!1}=W,j=[],X=0,x=0,T=0;try{let _=(Q||await N(J)).filter((w)=>w!=="shard_mappings");for(let w of _)try{let F=await p(J,w,V);if(!F.isValid){j.push(`Table ${w}: ${F.issues.join(", ")}`);continue}if(H){let z=await g(J,w,V);if(z.length===0){j.push(`Table ${w} has no records to process`);continue}if(!Y)for(let L of z){let D=String(L[V]),B=[];if(L.username&&typeof L.username==="string")B.push(`username:${L.username}`);if(L.email&&typeof L.email==="string")B.push(`email:${L.email}`);if(L.name&&typeof L.name==="string")B.push(`name:${L.name}`);await $.setShardMapping(D,Z,B),T++}x+=z.length}else{let z=await h(J,w,V);if(z.length===0){j.push(`Table ${w} has no records to process`);continue}if(!Y)for(let L of z)await $.setShardMapping(L,Z),T++;x+=z.length}X++}catch(F){j.push(`Failed to process table ${w}: ${F}`)}if(U&&!Y){if(!(await N(J)).includes("shard_mappings"))await J.prepare(`
|
|
2
2
|
CREATE TABLE IF NOT EXISTS shard_mappings (
|
|
3
3
|
primary_key TEXT PRIMARY KEY,
|
|
4
4
|
shard_name TEXT NOT NULL,
|
|
5
5
|
created_at INTEGER NOT NULL,
|
|
6
6
|
updated_at INTEGER NOT NULL
|
|
7
|
-
);`.trim()).run()}if(!
|
|
7
|
+
);`.trim()).run()}if(!Y)await $.addKnownShard(Z)}catch(A){j.push(`Integration failed: ${A}`)}return{success:j.length===0||j.length>0&&X>0,shardName:Z,tablesProcessed:X,totalRecords:x,mappingsCreated:T,issues:j}}async function VJ(J,Z,$,W={}){let{primaryKeyColumn:Q="id",tablesToCheck:V,skipCache:O=!1,maxRecordsToCheck:U=1000,migrateOtherColumns:Y=!1}=W,H=`${Z}_migration_check`;if(!O&&M.has(H))return{migrationNeeded:!1,migrationPerformed:!1,recordsMigrated:0,tablesProcessed:0,issues:[]};let j=[],X=0,x=0,T=!1,A=!1;try{let{KVShardMapper:_}=await Promise.resolve().then(() => (S(),n)),w=new _($.kv,{hashShardMappings:$.hashShardMappings}),F=await N(J),z=V||F.filter((L)=>L!=="shard_mappings"&&!L.startsWith("sqlite_")&&L!=="sqlite_sequence");if(z.length===0)return M.set(H,!0),{migrationNeeded:!1,migrationPerformed:!1,recordsMigrated:0,tablesProcessed:0,issues:[]};for(let L of z)try{let D=await p(J,L,Q);if(!D.isValid||D.recordCount===0)continue;let B=Math.min(U,D.recordCount),K=await J.prepare(`
|
|
8
8
|
SELECT ${Q} FROM ${L}
|
|
9
9
|
ORDER BY ${Q}
|
|
10
|
-
LIMIT ?`.trim()).bind(
|
|
10
|
+
LIMIT ?`.trim()).bind(B).all(),r=0,LJ=K.results.slice(0,10).map(async(C)=>{let P=String(C[Q]);return{key:P,mapping:await w.getShardMapping(P)}}),XJ=await Promise.all(LJ);for(let C of XJ)if(!C.mapping)r++,T=!0;if(r>0){if($.debug)console.log(`Auto-migrating table ${L} in shard ${Z} (${D.recordCount} records)`);if(Y){let C=await g(J,L,Q),P=0;for(let q of C){let l=String(q[Q]);if(!await w.getShardMapping(l)){let c=[];if(q.username&&typeof q.username==="string")c.push(`username:${q.username}`);if(q.email&&typeof q.email==="string")c.push(`email:${q.email}`);if(q.name&&typeof q.name==="string")c.push(`name:${q.name}`);await w.setShardMapping(l,Z,c),P++}}X+=P}else{let C=await h(J,L,Q),P=0;for(let q of C)if(!await w.getShardMapping(q))await w.setShardMapping(q,Z),P++;X+=P}if(x++,A=!0,$.debug)console.log(`Auto-migrated ${X} records from table ${L}`)}}catch(D){j.push(`Auto-migration failed for table ${L}: ${D}`)}if(A){if(await w.addKnownShard(Z),!F.includes("shard_mappings"))await J.prepare(`CREATE TABLE IF NOT EXISTS shard_mappings (
|
|
11
11
|
primary_key TEXT PRIMARY KEY,
|
|
12
12
|
shard_name TEXT NOT NULL,
|
|
13
13
|
created_at INTEGER NOT NULL,
|
|
14
14
|
updated_at INTEGER NOT NULL
|
|
15
15
|
);
|
|
16
|
-
`).run()}if(q.set(j,!0),D&&$.debug)console.log(`Auto-migration completed for shard ${Z}: ${x} records from ${F} tables`)}catch(S){Y.push(`Auto-migration error: ${S}`)}return{migrationNeeded:H,migrationPerformed:D,recordsMigrated:x,tablesProcessed:F,issues:Y}}async function QJ(J,Z,$){let V=`${Z}_migration_check`;if(q.has(V))return!1;try{let Q=await k(J);if(Q.includes("shard_mappings"))return q.set(V,!0),!1;let{KVShardMapper:O}=await Promise.resolve().then(() => (f(),l)),z=new O($.kv,{hashShardMappings:$.hashShardMappings}),U=Q.filter((j)=>j!=="shard_mappings"&&!j.startsWith("sqlite_")&&j!=="sqlite_sequence");for(let j of U.slice(0,3))try{if(((await J.prepare(`SELECT COUNT(*) as count FROM ${j} LIMIT 1`).first())?.count||0)>0){let F=await J.prepare(`SELECT id FROM ${j} LIMIT 1`).first();if(F){let H=String(F.id);if(!await z.getShardMapping(H))return!0}}}catch{continue}return!1}catch{return!1}}function VJ(){q.clear()}function WJ(J){let Z=`${J}_migration_check`;q.delete(Z)}var q;var M=y(()=>{C();q=new Map});C();f();var u=null;function xJ(J){if(u=J,J.shards&&Object.keys(J.shards).length>0&&!J.disableAutoMigration)UJ(J).catch((Z)=>{console.warn("Background auto-migration failed:",Z)})}async function OJ(J){if(u=J,J.shards&&Object.keys(J.shards).length>0&&!J.disableAutoMigration)try{await UJ(J)}catch(Z){console.warn("Auto migration failed:",Z)}}async function FJ(J,Z){return await OJ(J),await Z()}async function UJ(J){try{let{autoDetectAndMigrate:Z}=await Promise.resolve().then(() => (M(),b)),$=Object.keys(J.shards);console.log(`\uD83D\uDD0D Checking ${$.length} shards for existing data...`);let V=$.map(async(O)=>{let z=J.shards[O];if(!z)return null;try{let U=await Z(z,O,J,{maxRecordsToCheck:1000});return{shardName:O,...U}}catch(U){return console.warn(`Auto-migration failed for shard ${O}:`,U),null}}),W=(await Promise.all(V)).filter((O)=>O?.migrationPerformed);if(J.debug)if(W.length>0){let O=W.reduce((z,U)=>z+(U?.recordsMigrated||0),0);console.log(`\uD83C\uDF89 Auto-migration completed! Migrated ${O} records across ${W.length} shards`),W.forEach((z)=>{if(z)console.log(` ✅ ${z.shardName}: ${z.recordsMigrated} records from ${z.tablesProcessed} tables`)})}else console.log("✅ All shards ready - no migration needed")}catch(Z){console.warn("Background auto-migration setup failed:",Z)}}function GJ(){u=null}function A(){if(!u)throw new G("CollegeDB not initialized. Call initialize() first.","NOT_INITIALIZED");return u}function LJ(J){let Z=J.trim().toUpperCase();if(Z.startsWith("SELECT")||Z.startsWith("VALUES")||Z.startsWith("TABLE")||Z.startsWith("PRAGMA")||Z.startsWith("EXPLAIN")||Z.startsWith("WITH")||Z.startsWith("SHOW"))return"read";return"write"}function XJ(J,Z){let $=J.strategy||"hash";if(typeof $==="string")return $;return $[Z]}function HJ(J,Z){if(J===Z)return 0;let $={wnam:{lat:37.7749,lon:-122.4194},enam:{lat:40.7128,lon:-74.006},weur:{lat:51.5074,lon:-0.1278},eeur:{lat:52.52,lon:13.405},apac:{lat:35.6762,lon:139.6503},oc:{lat:-33.8688,lon:151.2093},me:{lat:25.2048,lon:55.2708},af:{lat:-26.2041,lon:28.0473}},V=$[J],Q=$[Z],W=V.lat-Q.lat,O=V.lon-Q.lon;return Math.sqrt(W*W+O*O)}function TJ(J){let Z=J.cf;if(!Z||!Z.country)return"wnam";let{country:$,continent:V}=Z;if(["US","CA","MX"].includes($)){let Q=Z.region||Z.regionCode||"",W=Z.timezone||"";if(Q.includes("CA")||Q.includes("WA")||Q.includes("OR")||Q.includes("NV")||Q.includes("AZ")||Q.includes("UT")||W.includes("Pacific")||W.includes("America/Los_Angeles"))return"wnam";return"enam"}if(["GL","PM","BM"].includes($))return"enam";if(["GB","IE","FR","ES","PT","NL","BE","LU","CH","AT","IT"].includes($))return"weur";if(["DE","PL","CZ","SK","HU","SI","HR","BA","RS","ME","MK","AL","BG","RO","MD","UA","BY","LT","LV","EE","FI","SE","NO","DK","IS"].includes($))return"eeur";if($==="RU")return"eeur";if(["JP","KR","CN","HK","TW","MO","MN","KP"].includes($))return"apac";if(["TH","VN","SG","MY","ID","PH","BN","KH","LA","MM","TL","IN","PK","BD","LK","NP","BT","MV","AF"].includes($))return"apac";if(["AU","NZ","PG","FJ","NC","VU","SB","WS","TO","KI","NR","PW","FM","MH","TV"].includes($))return"oc";if(["AE","SA","QA","KW","BH","OM","YE","IQ","IR","SY","LB","JO","IL","PS","TR","CY"].includes($))return"me";if(V==="AF"||["EG","LY","TN","DZ","MA","SD","SS","ET","ER","DJ","SO"].includes($))return"af";if(["KZ","UZ","TM","TJ","KG"].includes($))return"eeur";if(V==="SA"||["BR","AR","CL","PE","CO","VE","EC","BO","PY","UY","GY","SR","GF"].includes($))return"enam";if(["GT","BZ","SV","HN","NI","CR","PA","CU","JM","HT","DO","PR","TT","BB","GD","VC","LC","DM","AG","KN"].includes($))return"enam";return"wnam"}function wJ(J,Z,$,V){let Q=Z.filter((Y)=>$[Y]);if(Q.length===0){let Y=0;for(let F=0;F<V.length;F++){let H=V.charCodeAt(F);Y=(Y<<5)-Y+H,Y=Y&Y}let x=Math.abs(Y)%Z.length;return Z[x]}let W=Q.map((Y)=>{let x=$[Y],F=HJ(J,x.region),H=x.priority||1,D=F-H*0.1;return{shard:Y,score:D,distance:F,priority:H}});W.sort((Y,x)=>Y.score-x.score);let O=W[0].score,z=W.filter((Y)=>Math.abs(Y.score-O)<0.01);if(z.length===1)return z[0].shard;let U=0;for(let Y=0;Y<V.length;Y++){let x=V.charCodeAt(Y);U=(U<<5)-U+x,U=U&U}let j=Math.abs(U)%z.length;return z[j].shard}async function AJ(J,Z="write"){let $=A(),V=new E($.kv,{hashShardMappings:$.hashShardMappings}),Q=await V.getShardMapping(J);if(Q)return Q.shard;let W=Object.keys($.shards);if(W.length===0)throw new G("No shards configured","NO_SHARDS");for(let U of W){let j=$.shards[U];if(!j)continue;try{let{autoDetectAndMigrate:Y}=await Promise.resolve().then(() => (M(),b));if((await Y(j,U,$,{maxRecordsToCheck:100})).migrationPerformed){let F=await V.getShardMapping(J);if(F)return F.shard}}catch(Y){console.warn(`Auto-migration check failed for shard ${U}:`,Y)}}let O,z=XJ($,Z);if($.coordinator)try{let U=$.coordinator.idFromName("default"),Y=await $.coordinator.get(U).fetch("http://coordinator/allocate",{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({primaryKey:J,strategy:z,operationType:Z,targetRegion:$.targetRegion,shardLocations:$.shardLocations})});if(Y.ok)O=(await Y.json()).shard;else O=W[Math.floor(Math.random()*W.length)]}catch(U){console.warn("Coordinator allocation failed, falling back to local strategy:",U),O=W[Math.floor(Math.random()*W.length)]}else switch(z){case"hash":let U=0;for(let Y=0;Y<J.length;Y++){let x=J.charCodeAt(Y);U=(U<<5)-U+x,U=U&U}let j=Math.abs(U)%W.length;O=W[j]||W[0];break;case"location":if(!$.targetRegion){console.warn("Location strategy requires targetRegion in config, falling back to hash");let Y=0;for(let F=0;F<J.length;F++){let H=J.charCodeAt(F);Y=(Y<<5)-Y+H,Y=Y&Y}let x=Math.abs(Y)%W.length;O=W[x]||W[0]}else O=wJ($.targetRegion,W,$.shardLocations||{},J);break;case"random":O=W[Math.floor(Math.random()*W.length)]||W[0];break;default:O=W[0];break}return await V.setShardMapping(J,O),O}async function DJ(J,Z="write"){let $=A(),V=await AJ(J,Z),Q=$.shards[V];if(!Q)throw new G(`Shard ${V} not found in configuration`,"SHARD_NOT_FOUND");return Q}async function vJ(J,Z){let{createSchema:$}=await Promise.resolve().then(() => (M(),b));await $(J,Z)}async function p(J,Z){let $=LJ(Z);return(await DJ(J,$)).prepare(Z)}async function BJ(J,Z,$=[]){let Q=await(await p(J,Z)).bind(...$).run();if(!Q.success)throw new G(`Query failed: ${Q.error||"Unknown error"}`,"QUERY_FAILED");return Q}async function RJ(J,Z,$=[]){let Q=await(await p(J,Z)).bind(...$).all();if(!Q.success)throw new G(`Query failed: ${Q.error||"Unknown error"}`,"QUERY_FAILED");return Q}async function EJ(J,Z,$=[]){return await(await p(J,Z)).bind(...$).first()}async function IJ(J,Z,$){let V=A();if(!V.shards[Z])throw new G(`Shard ${Z} not found in configuration`,"SHARD_NOT_FOUND");let Q=new E(V.kv,{hashShardMappings:V.hashShardMappings}),W=await Q.getShardMapping(J);if(!W)throw new G(`No existing mapping found for primary key: ${J}`,"MAPPING_NOT_FOUND");if(W.shard!==Z){let{migrateRecord:O}=await Promise.resolve().then(() => (M(),b)),z=V.shards[W.shard],U=V.shards[Z];if(!z||!U)throw new G("Source or target shard not available","SHARD_UNAVAILABLE");await O(z,U,J,$)}await Q.updateShardMapping(J,Z)}async function _J(){let J=A();if(J.coordinator)try{let Z=J.coordinator.idFromName("default"),V=await J.coordinator.get(Z).fetch("http://coordinator/shards");if(V.ok)return await V.json()}catch(Z){console.warn("Failed to get shards from coordinator:",Z)}return Object.keys(J.shards)}async function qJ(){let J=A();if(J.coordinator)try{let V=J.coordinator.idFromName("default"),W=await J.coordinator.get(V).fetch("http://coordinator/stats");if(W.ok)return await W.json()}catch(V){console.warn("Failed to get stats from coordinator:",V)}let $=await new E(J.kv,{hashShardMappings:J.hashShardMappings}).getShardKeyCounts();return Object.entries(J.shards).map(([V,Q])=>({binding:V,count:$[V]||0}))}async function PJ(J,Z,$=[]){let Q=A().shards[J];if(!Q)throw new G(`Shard ${J} not found`,"SHARD_NOT_FOUND");let W=await Q.prepare(Z).bind(...$).run();if(!W.success)throw new G(`Query failed: ${W.error||"Unknown error"}`,"QUERY_FAILED");return W}async function CJ(J,Z,$=[]){let Q=A().shards[J];if(!Q)throw new G(`Shard ${J} not found`,"SHARD_NOT_FOUND");return await Q.prepare(Z).bind(...$).all()}async function kJ(J,Z,$=[]){let Q=A().shards[J];if(!Q)throw new G(`Shard ${J} not found`,"SHARD_NOT_FOUND");return await Q.prepare(Z).bind(...$).first()}async function MJ(J,Z=[],$=50){let V=A(),Q=[];for(let[O,z]of Object.entries(V.shards))try{let U=z.prepare(J).bind(...Z).all().catch((j)=>{return console.error(`Error executing query on shard ${O}:`,j),{success:!1,results:[],meta:{count:0,duration:0}}});Q.push(U)}catch(U){console.error(`Error running on shard ${O}:`,U)}let W=[];for(let O=0;O<Q.length;O+=$)W.push(...await Promise.all(Q.slice(O,O+$)));return W}async function SJ(J,Z=[],$=50){let V=A(),Q=[];for(let[O,z]of Object.entries(V.shards))try{let U=z.prepare(J).bind(...Z).all().catch((j)=>{return console.error(`Error executing query on shard ${O}:`,j),{success:!1,results:[],meta:{count:0,duration:0}}});Q.push(U)}catch(U){console.error(`Error running on shard ${O}:`,U)}let W=[];for(let O=0;O<Q.length;O+=$)W.push(...await Promise.all(Q.slice(O,O+$)));return W}async function NJ(J,Z=[],$=50){let V=A(),Q=[];for(let[O,z]of Object.entries(V.shards))try{let U=z.prepare(J).bind(...Z).first().catch((j)=>{return console.error(`Error executing query on shard ${O}:`,j),null});Q.push(U)}catch(U){console.error(`Error running on shard ${O}:`,U)}let W=[];for(let O=0;O<Q.length;O+=$)W.push(...await Promise.all(Q.slice(O,O+$)));return W}async function fJ(){let J=A();if(await new E(J.kv,{hashShardMappings:J.hashShardMappings}).clearAllMappings(),J.coordinator)try{let $=J.coordinator.idFromName("default");await J.coordinator.get($).fetch("http://coordinator/flush",{method:"POST"})}catch($){console.warn("Failed to flush coordinator:",$)}}C();class o{state;constructor(J){this.state=J}async getState(){return await this.state.storage.get("coordinator_state")||{knownShards:[],shardStats:{},strategy:"round-robin",roundRobinIndex:0}}async saveState(J){await this.state.storage.put("coordinator_state",J)}async fetch(J){let $=new URL(J.url).pathname,V=J.method;try{switch(`${V} ${$}`){case"GET /shards":return this.handleListShards();case"POST /shards":return this.handleAddShard(J);case"DELETE /shards":return this.handleRemoveShard(J);case"GET /stats":return this.handleGetStats();case"POST /stats":return this.handleUpdateStats(J);case"POST /allocate":return this.handleAllocateShard(J);case"POST /flush":return this.handleFlush();case"GET /health":return new Response("OK",{status:200});default:return new Response("Not Found",{status:404})}}catch(Q){return console.error("ShardCoordinator error:",Q),new Response("Internal Server Error",{status:500})}}async handleListShards(){let J=await this.getState();return new Response(JSON.stringify(J.knownShards),{headers:{"Content-Type":"application/json"}})}async handleAddShard(J){let{shard:Z}=await J.json();if(!Z||typeof Z!=="string")return new Response(JSON.stringify({error:"Missing or invalid shard parameter"}),{status:400,headers:{"Content-Type":"application/json"}});let $=await this.getState();if(!$.knownShards.includes(Z))$.knownShards.push(Z),$.shardStats[Z]={binding:Z,count:0,lastUpdated:Date.now()},await this.saveState($);return new Response(JSON.stringify({success:!0}),{headers:{"Content-Type":"application/json"}})}async handleRemoveShard(J){let{shard:Z}=await J.json();if(!Z||typeof Z!=="string")return new Response(JSON.stringify({error:"Missing or invalid shard parameter"}),{status:400,headers:{"Content-Type":"application/json"}});let $=await this.getState(),V=$.knownShards.indexOf(Z);if(V>-1){if($.knownShards.splice(V,1),delete $.shardStats[Z],$.roundRobinIndex>=$.knownShards.length)$.roundRobinIndex=0;await this.saveState($)}return new Response(JSON.stringify({success:!0}),{headers:{"Content-Type":"application/json"}})}async handleGetStats(){let J=await this.getState(),Z=Object.values(J.shardStats);return new Response(JSON.stringify(Z),{headers:{"Content-Type":"application/json"}})}async handleUpdateStats(J){let{shard:Z,count:$}=await J.json();if(!Z||typeof Z!=="string")return new Response(JSON.stringify({error:"Missing or invalid shard parameter"}),{status:400,headers:{"Content-Type":"application/json"}});if($===void 0||typeof $!=="number")return new Response(JSON.stringify({error:"Missing or invalid count parameter"}),{status:400,headers:{"Content-Type":"application/json"}});let V=await this.getState();if(V.shardStats[Z])V.shardStats[Z].count=$,V.shardStats[Z].lastUpdated=Date.now(),await this.saveState(V);return new Response(JSON.stringify({success:!0}),{headers:{"Content-Type":"application/json"}})}async handleAllocateShard(J){let{primaryKey:Z,strategy:$,operationType:V}=await J.json();if(!Z||typeof Z!=="string")return new Response(JSON.stringify({error:"Missing or invalid primaryKey parameter"}),{status:400,headers:{"Content-Type":"application/json"}});let Q=await this.getState();if(Q.knownShards.length===0)return new Response(JSON.stringify({error:"No shards available"}),{status:400,headers:{"Content-Type":"application/json"}});let W=this.resolveStrategy(Q.strategy,$,V||"write"),O=this.selectShard(Z,Q,W);if(W==="round-robin")Q.roundRobinIndex=(Q.roundRobinIndex+1)%Q.knownShards.length,await this.saveState(Q);return new Response(JSON.stringify({shard:O}),{headers:{"Content-Type":"application/json"}})}async handleFlush(){return await this.state.storage.deleteAll(),new Response(JSON.stringify({success:!0}),{headers:{"Content-Type":"application/json"}})}resolveStrategy(J,Z,$="write"){if(Z)return Z;if(typeof J==="string")return J;return J[$]}selectShard(J,Z,$){let V=Z.knownShards;if(V.length===0)throw new G("No shards available","NO_SHARDS");switch($){case"round-robin":return V[Z.roundRobinIndex]??V[0];case"random":return V[Math.floor(Math.random()*V.length)];case"hash":let Q=0;for(let U=0;U<J.length;U++){let j=J.charCodeAt(U);Q=(Q<<5)-Q+j,Q=Q&Q}let W=Math.abs(Q)%V.length;return V[W];case"location":let O=0;for(let U=0;U<J.length;U++){let j=J.charCodeAt(U);O=(O<<5)-O+j,O=O&O}let z=Math.abs(O)%V.length;return V[z];default:return V[0]}}async incrementShardCount(J){let Z=await this.getState();if(Z.shardStats[J])Z.shardStats[J].count++,Z.shardStats[J].lastUpdated=Date.now(),await this.saveState(Z)}async decrementShardCount(J){let Z=await this.getState();if(Z.shardStats[J]&&Z.shardStats[J].count>0)Z.shardStats[J].count--,Z.shardStats[J].lastUpdated=Date.now(),await this.saveState(Z)}}if(typeof global!=="undefined"){class J{data=new Map;async get(V){return this.data.get(V)}async put(V,Q){this.data.set(V,Q)}async delete(V){return this.data.delete(V)}async deleteAll(){this.data.clear()}async list(V){if(!V?.prefix)return new Map(this.data);let Q=new Map;for(let[W,O]of this.data.entries())if(W.startsWith(V.prefix))Q.set(W,O);return Q}}class Z{storage;constructor(){this.storage=new J}}class ${coordinator;mockState;constructor(){this.mockState=new Z,this.coordinator=new o(this.mockState)}async testShardAllocation(){await this.coordinator.fetch(new Request("http://test/shards",{method:"POST",body:JSON.stringify({shard:"db-east"})})),await this.coordinator.fetch(new Request("http://test/shards",{method:"POST",body:JSON.stringify({shard:"db-west"})}));let Q=await(await this.coordinator.fetch(new Request("http://test/allocate",{method:"POST",body:JSON.stringify({primaryKey:"user-1",strategy:"round-robin"})}))).json(),O=await(await this.coordinator.fetch(new Request("http://test/allocate",{method:"POST",body:JSON.stringify({primaryKey:"user-2",strategy:"round-robin"})}))).json();console.assert(Q.shard!==O.shard,"Round-robin should alternate shards");let U=await(await this.coordinator.fetch(new Request("http://test/allocate",{method:"POST",body:JSON.stringify({primaryKey:"consistent-key",strategy:"hash"})}))).json(),Y=await(await this.coordinator.fetch(new Request("http://test/allocate",{method:"POST",body:JSON.stringify({primaryKey:"consistent-key",strategy:"hash"})}))).json();console.assert(U.shard===Y.shard,"Hash allocation should be consistent"),console.log("✅ Shard allocation tests passed")}async testShardStats(){await this.coordinator.fetch(new Request("http://test/flush",{method:"POST"})),await this.coordinator.fetch(new Request("http://test/shards",{method:"POST",body:JSON.stringify({shard:"db-stats-test"})})),await this.coordinator.fetch(new Request("http://test/stats",{method:"POST",body:JSON.stringify({shard:"db-stats-test",count:42})}));let Q=await(await this.coordinator.fetch(new Request("http://test/stats",{method:"GET"}))).json();console.assert(Q.length===1,"Should have one shard stat"),console.assert(Q[0]?.binding==="db-stats-test","Should have correct binding name"),console.assert(Q[0]?.count===42,"Should have correct count"),console.log("✅ Shard stats tests passed")}async testErrorHandling(){await this.coordinator.fetch(new Request("http://test/flush",{method:"POST"}));let V=await this.coordinator.fetch(new Request("http://test/allocate",{method:"POST",body:JSON.stringify({primaryKey:"test-key"})}));console.assert(V.status===400,"Should return 400 for no shards available");let Q=await this.coordinator.fetch(new Request("http://test/invalid",{method:"GET"}));console.assert(Q.status===404,"Should return 404 for invalid endpoint"),console.log("✅ Error handling tests passed")}async testCountManagement(){await this.coordinator.fetch(new Request("http://test/shards",{method:"POST",body:JSON.stringify({shard:"db-count-test"})})),await this.coordinator.incrementShardCount("db-count-test"),await this.coordinator.incrementShardCount("db-count-test");let V=await this.coordinator.fetch(new Request("http://test/stats",{method:"GET"})),Q=await V.json(),W=Q.find((z)=>z.binding==="db-count-test");console.assert(W?.count===2,"Count should be 2 after two increments"),await this.coordinator.decrementShardCount("db-count-test"),V=await this.coordinator.fetch(new Request("http://test/stats",{method:"GET"})),Q=await V.json();let O=Q.find((z)=>z.binding==="db-count-test");console.assert(O?.count===1,"Count should be 1 after decrement"),console.log("✅ Count management tests passed")}async runAllTests(){console.log("\uD83E\uDDEA Running ShardCoordinator tests...");try{return await this.testShardAllocation(),await this.testShardStats(),await this.testErrorHandling(),await this.testCountManagement(),console.log("\uD83C\uDF89 All ShardCoordinator tests passed!"),!0}catch(V){return console.error("❌ ShardCoordinator tests failed:",V),!1}}}globalThis.testShardCoordinator=()=>new $}C();f();M();export{h as validateTableForSharding,n as schemaExists,PJ as runShard,MJ as runAllShards,BJ as run,GJ as resetConfig,IJ as reassignShard,p as prepare,e as migrateRecord,k as listTables,_J as listKnownShards,ZJ as integrateExistingDatabase,OJ as initializeAsync,xJ as initialize,qJ as getShardStats,TJ as getClosestRegionFromIP,fJ as flush,kJ as firstShard,NJ as firstAllShards,EJ as first,a as dropSchema,g as discoverExistingRecordsWithColumns,c as discoverExistingPrimaryKeys,r as createSchemaAcrossShards,vJ as createSchema,JJ as createMappingsForExistingKeys,FJ as collegedb,WJ as clearShardMigrationCache,VJ as clearMigrationCache,QJ as checkMigrationNeeded,$J as autoDetectAndMigrate,CJ as allShard,SJ as allAllShards,RJ as all,o as ShardCoordinator,E as KVShardMapper,G as CollegeDBError};
|
|
16
|
+
`).run()}if(M.set(H,!0),A&&$.debug)console.log(`Auto-migration completed for shard ${Z}: ${X} records from ${x} tables`)}catch(_){j.push(`Auto-migration error: ${_}`)}return{migrationNeeded:T,migrationPerformed:A,recordsMigrated:X,tablesProcessed:x,issues:j}}async function OJ(J,Z,$){let W=`${Z}_migration_check`;if(M.has(W))return!1;try{let Q=await N(J);if(Q.includes("shard_mappings"))return M.set(W,!0),!1;let{KVShardMapper:O}=await Promise.resolve().then(() => (S(),n)),U=new O($.kv,{hashShardMappings:$.hashShardMappings}),Y=Q.filter((H)=>H!=="shard_mappings"&&!H.startsWith("sqlite_")&&H!=="sqlite_sequence");for(let H of Y.slice(0,3))try{if(((await J.prepare(`SELECT COUNT(*) as count FROM ${H} LIMIT 1`).first())?.count||0)>0){let x=await J.prepare(`SELECT id FROM ${H} LIMIT 1`).first();if(x){let T=String(x.id);if(!await U.getShardMapping(T))return!0}}}catch{continue}return!1}catch{return!1}}function UJ(){M.clear()}function YJ(J){let Z=`${J}_migration_check`;M.delete(Z)}var M,o;var f=d(()=>{k();M=new Map,o=new Map});k();S();var b=null;function xJ(J){b=J;try{let Z=new I(J.kv,{hashShardMappings:J.hashShardMappings});Promise.all(Object.keys(J.shards).map(($)=>Z.addKnownShard($))).catch(()=>{return})}catch{}if(J.shards&&Object.keys(J.shards).length>0&&!J.disableAutoMigration)HJ(J).catch((Z)=>{console.warn("Background auto-migration failed:",Z)})}async function jJ(J){b=J;try{let Z=new I(J.kv,{hashShardMappings:J.hashShardMappings});await Promise.all(Object.keys(J.shards).map(($)=>Z.addKnownShard($)))}catch{}if(J.shards&&Object.keys(J.shards).length>0&&!J.disableAutoMigration)try{await HJ(J)}catch(Z){console.warn("Auto migration failed:",Z)}}async function zJ(J,Z){return await jJ(J),await Z()}async function HJ(J){try{let{autoDetectAndMigrate:Z}=await Promise.resolve().then(() => (f(),m)),$=Object.keys(J.shards);if(J.debug)console.log(`\uD83D\uDD0D Checking ${$.length} shards for existing data...`);let W=$.map(async(O)=>{let U=J.shards[O];if(!U)return null;try{let Y=await Z(U,O,J,{maxRecordsToCheck:1000});return{shardName:O,...Y}}catch(Y){return console.warn(`Auto-migration failed for shard ${O}:`,Y),null}}),V=(await Promise.all(W)).filter((O)=>O?.migrationPerformed);if(J.debug)if(V.length>0){let O=V.reduce((U,Y)=>U+(Y?.recordsMigrated||0),0);console.log(`\uD83C\uDF89 Auto-migration completed! Migrated ${O} records across ${V.length} shards`),V.forEach((U)=>{if(U)console.log(` ✅ ${U.shardName}: ${U.recordsMigrated} records from ${U.tablesProcessed} tables`)})}else console.log("✅ All shards ready - no migration needed")}catch(Z){console.warn("Background auto-migration setup failed:",Z)}}function TJ(){b=null}function E(){if(!b)throw new G("CollegeDB not initialized. Call initialize() first.","NOT_INITIALIZED");return b}function wJ(J){let Z=J.trim().toUpperCase();if(Z.startsWith("SELECT")||Z.startsWith("VALUES")||Z.startsWith("TABLE")||Z.startsWith("PRAGMA")||Z.startsWith("EXPLAIN")||Z.startsWith("WITH")||Z.startsWith("SHOW"))return"read";return"write"}function AJ(J,Z){let $=J.strategy||"hash";if(typeof $==="string")return $;let W=$;return W[Z]||W.write||W.read||"hash"}function DJ(J,Z){if(J===Z)return 0;let $={wnam:{lat:37.7749,lon:-122.4194},enam:{lat:40.7128,lon:-74.006},weur:{lat:51.5074,lon:-0.1278},eeur:{lat:52.52,lon:13.405},apac:{lat:35.6762,lon:139.6503},oc:{lat:-33.8688,lon:151.2093},me:{lat:25.2048,lon:55.2708},af:{lat:-26.2041,lon:28.0473}},W=$[J],Q=$[Z],V=W.lat-Q.lat,O=W.lon-Q.lon;return Math.sqrt(V*V+O*O)}function _J(J){let Z=J.cf;if(!Z||!Z.country)return"wnam";let{country:$,continent:W}=Z;if(["US","CA","MX"].includes($)){let Q=Z.region||Z.regionCode||"",V=Z.timezone||"";if(Q.includes("CA")||Q.includes("WA")||Q.includes("OR")||Q.includes("NV")||Q.includes("AZ")||Q.includes("UT")||V.includes("Pacific")||V.includes("America/Los_Angeles"))return"wnam";return"enam"}if(["GL","PM","BM"].includes($))return"enam";if(["GB","IE","FR","ES","PT","NL","BE","LU","CH","AT","IT"].includes($))return"weur";if(["DE","PL","CZ","SK","HU","SI","HR","BA","RS","ME","MK","AL","BG","RO","MD","UA","BY","LT","LV","EE","FI","SE","NO","DK","IS"].includes($))return"eeur";if($==="RU")return"eeur";if(["JP","KR","CN","HK","TW","MO","MN","KP"].includes($))return"apac";if(["TH","VN","SG","MY","ID","PH","BN","KH","LA","MM","TL","IN","PK","BD","LK","NP","BT","MV","AF"].includes($))return"apac";if(["AU","NZ","PG","FJ","NC","VU","SB","WS","TO","KI","NR","PW","FM","MH","TV"].includes($))return"oc";if(["AE","SA","QA","KW","BH","OM","YE","IQ","IR","SY","LB","JO","IL","PS","TR","CY"].includes($))return"me";if(W==="AF"||["EG","LY","TN","DZ","MA","SD","SS","ET","ER","DJ","SO"].includes($))return"af";if(["KZ","UZ","TM","TJ","KG"].includes($))return"eeur";if(W==="SA"||["BR","AR","CL","PE","CO","VE","EC","BO","PY","UY","GY","SR","GF"].includes($))return"enam";if(["GT","BZ","SV","HN","NI","CR","PA","CU","JM","HT","DO","PR","TT","BB","GD","VC","LC","DM","AG","KN"].includes($))return"enam";return"wnam"}function qJ(J){if(typeof J==="string")return J;return J.region||"wnam"}async function FJ(J){try{let[Z,$]=await Promise.all([J.prepare("PRAGMA page_count").first(),J.prepare("PRAGMA page_size").first()]);if(!Z?.page_count||!$?.page_size)throw new G("Failed to retrieve database size information","SIZE_QUERY_FAILED");return Z.page_count*$.page_size}catch(Z){throw new G(`Failed to get database size: ${Z instanceof Error?Z.message:"Unknown error"}`,"SIZE_QUERY_FAILED")}}async function EJ(J,Z){let $=Z.maxDatabaseSize||10683731148,Q=(await Promise.allSettled(J.map(async(V)=>{let O=Z.shards[V];if(!O)throw new G(`Shard ${V} not found in configuration`,"SHARD_NOT_FOUND");let U=await FJ(O);return{shard:V,size:U,withinLimit:U<$}}))).filter((V)=>V.status==="fulfilled"&&V.value.withinLimit).map((V)=>V.value.shard);if(Q.length===0){if(Z.debug)console.warn("All shards exceed maxDatabaseSize limit. Allowing allocation to prevent failure.");return J}if(Z.debug&&Q.length<J.length){let V=J.filter((O)=>!Q.includes(O));console.log(`Excluded ${V.length} shards due to size limits: ${V.join(", ")}`)}return Q}function RJ(J,Z,$,W){let Q=Z.filter((j)=>$[j]);if(Q.length===0){let j=0;for(let x=0;x<W.length;x++){let T=W.charCodeAt(x);j=(j<<5)-j+T,j=j&j}let X=Math.abs(j)%Z.length;return Z[X]}let V=Q.map((j)=>{let X=$[j],x=DJ(J,qJ(X)),T=typeof X==="object"?X.priority||1:1,A=x-T*0.1;return{shard:j,score:A,distance:x,priority:T}});V.sort((j,X)=>j.score-X.score);let O=V[0].score,U=V.filter((j)=>Math.abs(j.score-O)<0.01);if(U.length===1)return U[0].shard;let Y=0;for(let j=0;j<W.length;j++){let X=W.charCodeAt(j);Y=(Y<<5)-Y+X,Y=Y&Y}let H=Math.abs(Y)%U.length;return U[H].shard}function u(J,Z,$,W){switch(J){case"hash":{let Q=0;for(let O=0;O<Z.length;O++){let U=Z.charCodeAt(O);Q=(Q<<5)-Q+U,Q=Q&Q}let V=Math.abs(Q)%$.length;return $[V]||$[0]}case"location":{if(!W.targetRegion)return u("hash",Z,$,W);return RJ(W.targetRegion,$,W.shardLocations||{},Z)}case"random":return $[Math.floor(Math.random()*$.length)]||$[0];default:return u("hash",Z,$,W)}}async function IJ(J,Z="write"){let $=E(),W=new I($.kv,{hashShardMappings:$.hashShardMappings}),Q=await W.getShardMapping(J);if(Q)return Q.shard;let V=Object.keys($.shards);if(V.length===0)throw new G("No shards configured","NO_SHARDS");let O=await EJ(V,$),U,Y=AJ($,Z);if($.coordinator)try{let H=$.coordinator.idFromName("default"),X=await $.coordinator.get(H).fetch("http://coordinator/allocate",{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({primaryKey:J,strategy:Y,operationType:Z,targetRegion:$.targetRegion,shardLocations:$.shardLocations,availableShards:O})});if(X.ok)U=(await X.json()).shard;else U=u(Y,J,O,$)}catch(H){console.warn("Coordinator allocation failed, falling back to local strategy:",H),U=u(Y,J,O,$)}else U=u(Y,J,O,$);return await W.setShardMapping(J,U),U}async function BJ(J,Z="write"){let $=E(),W=await IJ(J,Z),Q=$.shards[W];if(!Q)throw new G(`Shard ${W} not found in configuration`,"SHARD_NOT_FOUND");return Q}async function vJ(J,Z){let{createSchema:$}=await Promise.resolve().then(() => (f(),m));await $(J,Z)}async function y(J,Z){let $=wJ(Z);return(await BJ(J,$)).prepare(Z)}async function PJ(J,Z,$=[]){let Q=await(await y(J,Z)).bind(...$).run();if(!Q.success)throw new G(`Query failed: ${Q.error||"Unknown error"}`,"QUERY_FAILED");return Q}async function CJ(J,Z,$=[]){let Q=await(await y(J,Z)).bind(...$).all();if(!Q.success)throw new G(`Query failed: ${Q.error||"Unknown error"}`,"QUERY_FAILED");return Q}async function MJ(J,Z,$=[]){return await(await y(J,Z)).bind(...$).first()}async function kJ(J,Z,$){let W=E();if(!W.shards[Z])throw new G(`Shard ${Z} not found in configuration`,"SHARD_NOT_FOUND");let Q=new I(W.kv,{hashShardMappings:W.hashShardMappings}),V=await Q.getShardMapping(J);if(!V)throw new G(`No existing mapping found for primary key: ${J}`,"MAPPING_NOT_FOUND");if(V.shard!==Z){let{migrateRecord:O}=await Promise.resolve().then(() => (f(),m)),U=W.shards[V.shard],Y=W.shards[Z];if(!U||!Y)throw new G("Source or target shard not available","SHARD_UNAVAILABLE");await O(U,Y,J,$)}await Q.updateShardMapping(J,Z)}async function NJ(){let J=E();if(J.coordinator)try{let Z=J.coordinator.idFromName("default"),W=await J.coordinator.get(Z).fetch("http://coordinator/shards");if(W.ok)return await W.json()}catch(Z){console.warn("Failed to get shards from coordinator:",Z)}try{let $=await new I(J.kv,{hashShardMappings:J.hashShardMappings}).getKnownShards(),W=new Set([...Object.keys(J.shards),...$]);return Array.from(W)}catch{return Object.keys(J.shards)}}async function SJ(){let J=E();if(J.coordinator)try{let Q=J.coordinator.idFromName("default"),O=await J.coordinator.get(Q).fetch("http://coordinator/stats");if(O.ok)return await O.json()}catch(Q){console.warn("Failed to get stats from coordinator:",Q)}let Z=new I(J.kv,{hashShardMappings:J.hashShardMappings}),$=await Z.getShardKeyCounts(),W=Object.keys(J.shards);try{let Q=await Z.getKnownShards();W=Array.from(new Set([...W,...Q]))}catch{}return W.map((Q)=>({binding:Q,count:$[Q]||0}))}async function fJ(J,Z,$=[]){let Q=E().shards[J];if(!Q)throw new G(`Shard ${J} not found`,"SHARD_NOT_FOUND");let V=await Q.prepare(Z).bind(...$).run();if(!V.success)throw new G(`Query failed: ${V.error||"Unknown error"}`,"QUERY_FAILED");return V}async function uJ(J,Z,$=[]){let Q=E().shards[J];if(!Q)throw new G(`Shard ${J} not found`,"SHARD_NOT_FOUND");return await Q.prepare(Z).bind(...$).all()}async function bJ(J,Z,$=[]){let Q=E().shards[J];if(!Q)throw new G(`Shard ${J} not found`,"SHARD_NOT_FOUND");return await Q.prepare(Z).bind(...$).first()}async function KJ(J,Z=[],$=50){let W=E(),Q=[];for(let[O,U]of Object.entries(W.shards)){if(!O||!U){console.error(`Shard ${O??"<null>"} not found, skipping`);continue}Q.push(()=>U.prepare(J).bind(...Z).run().catch((Y)=>{return console.error(`Error executing query on shard ${O}:`,Y),{success:!1,results:[],meta:{count:0,duration:0}}}))}let V=[];for(let O=0;O<Q.length;O+=$){let U=Q.slice(O,O+$).map((Y)=>Y());V.push(...await Promise.all(U))}return V}async function cJ(J,Z=[],$=50){let W=E(),Q=[];for(let[O,U]of Object.entries(W.shards)){if(!O||!U){console.error(`Shard ${O??"<null>"} not found, skipping`);continue}Q.push(()=>U.prepare(J).bind(...Z).all().catch((Y)=>{return console.error(`Error executing query on shard ${O}:`,Y),{success:!1,results:[],meta:{count:0,duration:0}}}))}let V=[];for(let O=0;O<Q.length;O+=$){let U=Q.slice(O,O+$).map((Y)=>Y());V.push(...await Promise.all(U))}return V}async function hJ(J,Z=[],$=50){let W=E(),Q=[];for(let[O,U]of Object.entries(W.shards)){if(!O||!U){console.error(`Shard ${O??"<null>"} not found, skipping`);continue}Q.push(()=>U.prepare(J).bind(...Z).first().catch((Y)=>{return console.error(`Error executing query on shard ${O}:`,Y),null}))}let V=[];for(let O=0;O<Q.length;O+=$){let U=Q.slice(O,O+$).map((Y)=>Y());V.push(...await Promise.all(U))}return V}async function gJ(){let J=E();if(await new I(J.kv,{hashShardMappings:J.hashShardMappings}).clearAllMappings(),J.coordinator)try{let $=J.coordinator.idFromName("default");await J.coordinator.get($).fetch("http://coordinator/flush",{method:"POST"})}catch($){console.warn("Failed to flush coordinator:",$)}}async function pJ(J){let $=E().shards[J];if(!$)throw new G(`Shard ${J} not found`,"SHARD_NOT_FOUND");return await FJ($)}k();class s{state;constructor(J){this.state=J}async getState(){return await this.state.storage.get("coordinator_state")||{knownShards:[],shardStats:{},strategy:"round-robin",roundRobinIndex:0}}async saveState(J){await this.state.storage.put("coordinator_state",J)}async fetch(J){let $=new URL(J.url).pathname,W=J.method;try{switch(`${W} ${$}`){case"GET /shards":return this.handleListShards();case"POST /shards":return this.handleAddShard(J);case"DELETE /shards":return this.handleRemoveShard(J);case"GET /stats":return this.handleGetStats();case"POST /stats":return this.handleUpdateStats(J);case"POST /allocate":return this.handleAllocateShard(J);case"POST /flush":return this.handleFlush();case"GET /health":return new Response("OK",{status:200});default:return new Response("Not Found",{status:404})}}catch(Q){return console.error("ShardCoordinator error:",Q),new Response("Internal Server Error",{status:500})}}async handleListShards(){let J=await this.getState();return new Response(JSON.stringify(J.knownShards),{headers:{"Content-Type":"application/json"}})}async handleAddShard(J){let{shard:Z}=await J.json();if(!Z||typeof Z!=="string")return new Response(JSON.stringify({error:"Missing or invalid shard parameter"}),{status:400,headers:{"Content-Type":"application/json"}});let $=await this.getState();if(!$.knownShards.includes(Z))$.knownShards.push(Z),$.shardStats[Z]={binding:Z,count:0,lastUpdated:Date.now()},await this.saveState($);return new Response(JSON.stringify({success:!0}),{headers:{"Content-Type":"application/json"}})}async handleRemoveShard(J){let{shard:Z}=await J.json();if(!Z||typeof Z!=="string")return new Response(JSON.stringify({error:"Missing or invalid shard parameter"}),{status:400,headers:{"Content-Type":"application/json"}});let $=await this.getState(),W=$.knownShards.indexOf(Z);if(W>-1){if($.knownShards.splice(W,1),delete $.shardStats[Z],$.roundRobinIndex>=$.knownShards.length)$.roundRobinIndex=0;await this.saveState($)}return new Response(JSON.stringify({success:!0}),{headers:{"Content-Type":"application/json"}})}async handleGetStats(){let J=await this.getState(),Z=Object.values(J.shardStats);return new Response(JSON.stringify(Z),{headers:{"Content-Type":"application/json"}})}async handleUpdateStats(J){let{shard:Z,count:$}=await J.json();if(!Z||typeof Z!=="string")return new Response(JSON.stringify({error:"Missing or invalid shard parameter"}),{status:400,headers:{"Content-Type":"application/json"}});if($===void 0||typeof $!=="number")return new Response(JSON.stringify({error:"Missing or invalid count parameter"}),{status:400,headers:{"Content-Type":"application/json"}});let W=await this.getState();if(W.shardStats[Z])W.shardStats[Z].count=$,W.shardStats[Z].lastUpdated=Date.now(),await this.saveState(W);return new Response(JSON.stringify({success:!0}),{headers:{"Content-Type":"application/json"}})}async handleAllocateShard(J){let{primaryKey:Z,strategy:$,operationType:W,availableShards:Q}=await J.json();if(!Z||typeof Z!=="string")return new Response(JSON.stringify({error:"Missing or invalid primaryKey parameter"}),{status:400,headers:{"Content-Type":"application/json"}});let V=await this.getState(),O=Q||V.knownShards;if(O.length===0)return new Response(JSON.stringify({error:"No shards available"}),{status:400,headers:{"Content-Type":"application/json"}});let U=this.resolveStrategy(V.strategy,$,W||"write"),Y=this.selectShard(Z,V,U,O);if(U==="round-robin")V.roundRobinIndex=(V.roundRobinIndex+1)%O.length,await this.saveState(V);return new Response(JSON.stringify({shard:Y}),{headers:{"Content-Type":"application/json"}})}async handleFlush(){return await this.state.storage.deleteAll(),new Response(JSON.stringify({success:!0}),{headers:{"Content-Type":"application/json"}})}resolveStrategy(J,Z,$="write"){if(Z)return Z;if(typeof J==="string")return J;return J[$]}selectShard(J,Z,$,W){let Q=W||Z.knownShards;if(Q.length===0)throw new G("No shards available","NO_SHARDS");switch($){case"round-robin":return Q[Z.roundRobinIndex]??Q[0];case"random":return Q[Math.floor(Math.random()*Q.length)];case"hash":{let V=0;for(let U=0;U<J.length;U++){let Y=J.charCodeAt(U);V=(V<<5)-V+Y,V=V&V}let O=Math.abs(V)%Q.length;return Q[O]}case"location":{let V=Z.targetRegion,O=Z.shardLocations||{},U=Q.filter((F)=>O[F]);if(!V||U.length===0){let F=0;for(let L=0;L<J.length;L++){let D=J.charCodeAt(L);F=(F<<5)-F+D,F=F&F}let z=Math.abs(F)%Q.length;return Q[z]}let Y={wnam:{lat:37.7749,lon:-122.4194},enam:{lat:40.7357,lon:-74.1724},weur:{lat:51.5074,lon:-0.1278},eeur:{lat:52.2297,lon:21.0122},apac:{lat:35.6762,lon:139.6503},oc:{lat:-33.8688,lon:151.2093},me:{lat:25.2048,lon:55.2708},af:{lat:-26.2041,lon:28.0473}},H=(F,z)=>(z in F),j=(F)=>H(Y,F)?F:"wnam",X=(F,z)=>{let L=Y[j(F)],D=Y[j(z)],B=L.lat-D.lat,K=L.lon-D.lon;return Math.sqrt(B*B+K*K)},x=U.map((F)=>{let z=O[F],L=X(V,z.region),D=z.priority||1;return{shard:F,score:L-D*0.1}});x.sort((F,z)=>F.score-z.score);let T=x[0].score,A=x.filter((F)=>Math.abs(F.score-T)<0.01);if(A.length===1)return A[0].shard;let _=0;for(let F=0;F<J.length;F++){let z=J.charCodeAt(F);_=(_<<5)-_+z,_=_&_}let w=Math.abs(_)%A.length;return A[w].shard}default:return Q[0]}}async incrementShardCount(J){let Z=await this.getState();if(Z.shardStats[J])Z.shardStats[J].count++,Z.shardStats[J].lastUpdated=Date.now(),await this.saveState(Z)}async decrementShardCount(J){let Z=await this.getState();if(Z.shardStats[J]&&Z.shardStats[J].count>0)Z.shardStats[J].count--,Z.shardStats[J].lastUpdated=Date.now(),await this.saveState(Z)}}if(typeof global!=="undefined"){class J{data=new Map;async get(W){return this.data.get(W)}async put(W,Q){this.data.set(W,Q)}async delete(W){return this.data.delete(W)}async deleteAll(){this.data.clear()}async list(W){if(!W?.prefix)return new Map(this.data);let Q=new Map;for(let[V,O]of this.data.entries())if(V.startsWith(W.prefix))Q.set(V,O);return Q}}class Z{storage;constructor(){this.storage=new J}}class ${coordinator;mockState;constructor(){this.mockState=new Z,this.coordinator=new s(this.mockState)}async testShardAllocation(){await this.coordinator.fetch(new Request("http://test/shards",{method:"POST",body:JSON.stringify({shard:"db-east"})})),await this.coordinator.fetch(new Request("http://test/shards",{method:"POST",body:JSON.stringify({shard:"db-west"})}));let Q=await(await this.coordinator.fetch(new Request("http://test/allocate",{method:"POST",body:JSON.stringify({primaryKey:"user-1",strategy:"round-robin"})}))).json(),O=await(await this.coordinator.fetch(new Request("http://test/allocate",{method:"POST",body:JSON.stringify({primaryKey:"user-2",strategy:"round-robin"})}))).json();console.assert(Q.shard!==O.shard,"Round-robin should alternate shards");let Y=await(await this.coordinator.fetch(new Request("http://test/allocate",{method:"POST",body:JSON.stringify({primaryKey:"consistent-key",strategy:"hash"})}))).json(),j=await(await this.coordinator.fetch(new Request("http://test/allocate",{method:"POST",body:JSON.stringify({primaryKey:"consistent-key",strategy:"hash"})}))).json();console.assert(Y.shard===j.shard,"Hash allocation should be consistent"),console.log("✅ Shard allocation tests passed")}async testShardStats(){await this.coordinator.fetch(new Request("http://test/flush",{method:"POST"})),await this.coordinator.fetch(new Request("http://test/shards",{method:"POST",body:JSON.stringify({shard:"db-stats-test"})})),await this.coordinator.fetch(new Request("http://test/stats",{method:"POST",body:JSON.stringify({shard:"db-stats-test",count:42})}));let Q=await(await this.coordinator.fetch(new Request("http://test/stats",{method:"GET"}))).json();console.assert(Q.length===1,"Should have one shard stat"),console.assert(Q[0]?.binding==="db-stats-test","Should have correct binding name"),console.assert(Q[0]?.count===42,"Should have correct count"),console.log("✅ Shard stats tests passed")}async testErrorHandling(){await this.coordinator.fetch(new Request("http://test/flush",{method:"POST"}));let W=await this.coordinator.fetch(new Request("http://test/allocate",{method:"POST",body:JSON.stringify({primaryKey:"test-key"})}));console.assert(W.status===400,"Should return 400 for no shards available");let Q=await this.coordinator.fetch(new Request("http://test/invalid",{method:"GET"}));console.assert(Q.status===404,"Should return 404 for invalid endpoint"),console.log("✅ Error handling tests passed")}async testCountManagement(){await this.coordinator.fetch(new Request("http://test/shards",{method:"POST",body:JSON.stringify({shard:"db-count-test"})})),await this.coordinator.incrementShardCount("db-count-test"),await this.coordinator.incrementShardCount("db-count-test");let W=await this.coordinator.fetch(new Request("http://test/stats",{method:"GET"})),Q=await W.json(),V=Q.find((U)=>U.binding==="db-count-test");console.assert(V?.count===2,"Count should be 2 after two increments"),await this.coordinator.decrementShardCount("db-count-test"),W=await this.coordinator.fetch(new Request("http://test/stats",{method:"GET"})),Q=await W.json();let O=Q.find((U)=>U.binding==="db-count-test");console.assert(O?.count===1,"Count should be 1 after decrement"),console.log("✅ Count management tests passed")}async runAllTests(){console.log("\uD83E\uDDEA Running ShardCoordinator tests...");try{return await this.testShardAllocation(),await this.testShardStats(),await this.testErrorHandling(),await this.testCountManagement(),console.log("\uD83C\uDF89 All ShardCoordinator tests passed!"),!0}catch(W){return console.error("❌ ShardCoordinator tests failed:",W),!1}}}globalThis.testShardCoordinator=()=>new $}k();S();f();export{p as validateTableForSharding,i as schemaExists,fJ as runShard,KJ as runAllShards,PJ as run,TJ as resetConfig,kJ as reassignShard,y as prepare,$J as migrateRecord,N as listTables,NJ as listKnownShards,WJ as integrateExistingDatabase,jJ as initializeAsync,xJ as initialize,SJ as getShardStats,pJ as getDatabaseSizeForShard,_J as getClosestRegionFromIP,gJ as flush,bJ as firstShard,hJ as firstAllShards,MJ as first,ZJ as dropSchema,g as discoverExistingRecordsWithColumns,h as discoverExistingPrimaryKeys,JJ as createSchemaAcrossShards,vJ as createSchema,QJ as createMappingsForExistingKeys,zJ as collegedb,YJ as clearShardMigrationCache,UJ as clearMigrationCache,OJ as checkMigrationNeeded,VJ as autoDetectAndMigrate,uJ as allShard,cJ as allAllShards,CJ as all,s as ShardCoordinator,I as KVShardMapper,G as CollegeDBError};
|
|
17
17
|
|
|
18
|
-
//# debugId=
|
|
18
|
+
//# debugId=60113EC501E41FC764756E2164756E21
|
|
19
19
|
//# sourceMappingURL=index.js.map
|