@earth-app/collegedb 1.1.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +294 -85
- package/dist/errors.d.ts +1 -1
- package/dist/index.d.ts +7 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -8
- package/dist/index.js.map +9 -9
- package/dist/kvmap.d.ts +1 -1
- package/dist/kvmap.d.ts.map +1 -1
- package/dist/migrations.d.ts +5 -5
- package/dist/migrations.d.ts.map +1 -1
- package/dist/providers.d.ts +78 -1
- package/dist/providers.d.ts.map +1 -1
- package/dist/router.d.ts +1 -1
- package/dist/router.d.ts.map +1 -1
- package/dist/types.d.ts +1 -1
- package/package.json +7 -8
package/README.md
CHANGED
|
@@ -8,7 +8,32 @@ Universal Database Horizontal Sharding Router
|
|
|
8
8
|
[](LICENSE)
|
|
9
9
|

|
|
10
10
|
|
|
11
|
-
A TypeScript library for **true horizontal scaling** of SQLite-style databases primarily for Cloudflare using D1 and KV, with additional provider adapters for Redis/Valkey KV and PostgreSQL/MySQL/SQLite SQL backends. CollegeDB distributes your data across multiple database shards, with each table's records split by primary key across different database instances.
|
|
11
|
+
A TypeScript library for **true horizontal scaling** of SQLite-style databases primarily for Cloudflare using D1 and KV, with additional provider adapters for Redis/Valkey KV and PostgreSQL/MySQL/SQLite SQL backends, plus Drizzle ORM interop across those SQL providers. CollegeDB distributes your data across multiple database shards, with each table's records split by primary key across different database instances.
|
|
12
|
+
|
|
13
|
+
## Table of Contents
|
|
14
|
+
|
|
15
|
+
- [Why CollegeDB](#why-collegedb)
|
|
16
|
+
- [Features](#features)
|
|
17
|
+
- [Getting Started](#getting-started)
|
|
18
|
+
- [Benchmark Suite](#benchmark-suite)
|
|
19
|
+
- [Provider Adapters](#provider-adapters)
|
|
20
|
+
- [NuxtHub + Drizzle Recipes](#nuxthub--drizzle-recipes)
|
|
21
|
+
- [Sandbox Benchmarks (Docker Compose)](#sandbox-benchmarks-docker-compose)
|
|
22
|
+
- [Basic Usage](#basic-usage)
|
|
23
|
+
- [Multi-Key Shard Mappings](#multi-key-shard-mappings)
|
|
24
|
+
- [Drop-in Replacement for Existing Databases](#drop-in-replacement-for-existing-databases)
|
|
25
|
+
- [Troubleshooting](#troubleshooting)
|
|
26
|
+
- [API Reference](#api-reference)
|
|
27
|
+
- [Architecture](#architecture)
|
|
28
|
+
- [Cloudflare Setup](#cloudflare-setup)
|
|
29
|
+
- [Monitoring and Maintenance](#monitoring-and-maintenance)
|
|
30
|
+
- [Performance Analysis](#performance-analysis)
|
|
31
|
+
- [Advanced Configuration](#advanced-configuration)
|
|
32
|
+
- [Quick Reference](#quick-reference)
|
|
33
|
+
- [Contributing](#contributing)
|
|
34
|
+
- [License](#license)
|
|
35
|
+
|
|
36
|
+
## Why CollegeDB
|
|
12
37
|
|
|
13
38
|
CollegeDB implements **data distribution** where a single logical table is physically stored across multiple D1 databases:
|
|
14
39
|
|
|
@@ -39,36 +64,132 @@ This allows you to:
|
|
|
39
64
|
- **Scale geographically** by placing shards in different regions
|
|
40
65
|
- **Increase write throughput** by parallelizing across multiple database instances
|
|
41
66
|
|
|
42
|
-
##
|
|
67
|
+
## Features
|
|
43
68
|
|
|
44
|
-
|
|
69
|
+
- Automatic query routing (primary key to shard mapping)
|
|
70
|
+
- Provider adapters for Redis/Valkey/NuxtHub KV plus PostgreSQL/MySQL/SQLite SQL
|
|
71
|
+
- Drizzle interop through existing SQL providers (`createPostgreSQLProvider`, `createMySQLProvider`, `createSQLiteProvider`)
|
|
72
|
+
- Hyperdrive helpers for PostgreSQL and MySQL
|
|
73
|
+
- Multiple allocation strategies: round-robin, random, hash, location-aware, and mixed read/write strategies
|
|
74
|
+
- Durable Object shard coordination and shard statistics
|
|
75
|
+
- Migration helpers for integrating existing datasets and rebalancing mappings
|
|
45
76
|
|
|
46
|
-
|
|
47
|
-
- **Route queries automatically** based on primary key mappings
|
|
48
|
-
- **Maintain consistency** with KV-based shard mapping
|
|
49
|
-
- **Run on multiple providers** through `KVStorage` and `SQLDatabase` contracts
|
|
50
|
-
- **Optimize for geography** with location-aware shard allocation
|
|
51
|
-
- **Monitor and rebalance** shard distribution
|
|
52
|
-
- **Handle migrations** between shards seamlessly
|
|
77
|
+
## Getting Started
|
|
53
78
|
|
|
54
|
-
|
|
79
|
+
### Installation
|
|
55
80
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
81
|
+
```bash
|
|
82
|
+
bun add @earth-app/collegedb
|
|
83
|
+
# or
|
|
84
|
+
npm install @earth-app/collegedb
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
### NuxtHub + Drizzle with CollegeDB Routing
|
|
88
|
+
|
|
89
|
+
Keep NuxtHub + Drizzle for schema/migrations and add CollegeDB as your routing layer.
|
|
90
|
+
|
|
91
|
+
```typescript
|
|
92
|
+
import { db as hubDb } from '@nuxthub/db';
|
|
93
|
+
import { kv } from '@nuxthub/kv';
|
|
94
|
+
import { sql } from 'drizzle-orm';
|
|
95
|
+
import { drizzle } from 'drizzle-orm/d1';
|
|
96
|
+
import { createNuxtHubKVProvider, createSQLiteProvider, first, initialize, run } from '@earth-app/collegedb';
|
|
97
|
+
|
|
98
|
+
let initialized = false;
|
|
99
|
+
|
|
100
|
+
function ensureCollegeDB(env: { DB_SECONDARY: D1Database }) {
|
|
101
|
+
if (initialized) return;
|
|
102
|
+
|
|
103
|
+
initialize({
|
|
104
|
+
kv: createNuxtHubKVProvider(kv),
|
|
105
|
+
shards: {
|
|
106
|
+
'db-primary': createSQLiteProvider(hubDb, sql),
|
|
107
|
+
'db-secondary': createSQLiteProvider(drizzle(env.DB_SECONDARY), sql)
|
|
108
|
+
},
|
|
109
|
+
strategy: 'hash'
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
initialized = true;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export default defineEventHandler(async (event) => {
|
|
116
|
+
const env = event.context.cloudflare.env;
|
|
117
|
+
ensureCollegeDB(env);
|
|
118
|
+
|
|
119
|
+
await run('post:123', 'INSERT OR REPLACE INTO blog_posts (id, title) VALUES (?, ?)', ['post:123', 'Hello from CollegeDB']);
|
|
120
|
+
|
|
121
|
+
const post = await first<{ id: string; title: string }>('post:123', 'SELECT id, title FROM blog_posts WHERE id = ?', ['post:123']);
|
|
122
|
+
|
|
123
|
+
return { post };
|
|
124
|
+
});
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
### Drop-in Pattern for Existing `hub:db` + `hub:kv` Code
|
|
128
|
+
|
|
129
|
+
```typescript
|
|
130
|
+
// before
|
|
131
|
+
import { eq } from 'drizzle-orm';
|
|
132
|
+
import { db } from 'hub:db';
|
|
133
|
+
import { kv } from 'hub:kv';
|
|
134
|
+
import { blogPosts } from '~/server/db/schema';
|
|
135
|
+
|
|
136
|
+
const cached = await kv.get('nuxtpress:post:slug');
|
|
137
|
+
if (cached) return cached;
|
|
138
|
+
|
|
139
|
+
const rows = await db.select().from(blogPosts).where(eq(blogPosts.slug, slug)).limit(1);
|
|
140
|
+
await kv.set('nuxtpress:post:slug', rows[0], { ttl: 3600 });
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
```typescript
|
|
144
|
+
// after (CollegeDB routing + same NuxtHub KV cache)
|
|
145
|
+
import { kv } from '@nuxthub/kv';
|
|
146
|
+
import { sql } from 'drizzle-orm';
|
|
147
|
+
import { db } from 'hub:db';
|
|
148
|
+
import { createNuxtHubKVProvider, createSQLiteProvider, first, initialize } from '@earth-app/collegedb';
|
|
149
|
+
|
|
150
|
+
let initialized = false;
|
|
151
|
+
|
|
152
|
+
function setup() {
|
|
153
|
+
if (initialized) return;
|
|
154
|
+
initialize({
|
|
155
|
+
kv: createNuxtHubKVProvider(kv),
|
|
156
|
+
shards: {
|
|
157
|
+
'db-primary': createSQLiteProvider(db, sql)
|
|
158
|
+
},
|
|
159
|
+
strategy: 'hash'
|
|
160
|
+
});
|
|
161
|
+
initialized = true;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
setup();
|
|
165
|
+
|
|
166
|
+
const cacheKey = `nuxtpress:post:${slug}`;
|
|
167
|
+
const cached = await kv.get(cacheKey);
|
|
168
|
+
if (cached) return cached;
|
|
169
|
+
|
|
170
|
+
const row = await first<{ id: string; slug: string; title: string }>(
|
|
171
|
+
cacheKey,
|
|
172
|
+
'SELECT id, slug, title FROM blog_posts WHERE slug = ? LIMIT 1',
|
|
173
|
+
[slug]
|
|
174
|
+
);
|
|
175
|
+
|
|
176
|
+
await kv.set(cacheKey, row, { ttl: 3600 });
|
|
177
|
+
```
|
|
66
178
|
|
|
67
179
|
## Benchmark Suite
|
|
68
180
|
|
|
69
|
-
CollegeDB includes a
|
|
181
|
+
CollegeDB includes a benchmark runner that executes each SQL+KV combination across adapter profiles, then generates a report with profile-specific matrices.
|
|
182
|
+
|
|
183
|
+
### Adapter Profiles
|
|
184
|
+
|
|
185
|
+
| Profile | Purpose |
|
|
186
|
+
| ---------- | ----------------------------------------------------------------------- |
|
|
187
|
+
| native | Direct provider clients (Cloudflare bindings or driver-native adapters) |
|
|
188
|
+
| drizzle | Drizzle interop through SQL provider adapters |
|
|
189
|
+
| hyperdrive | Hyperdrive connection-string wrappers for PostgreSQL/MySQL |
|
|
190
|
+
| nuxthub | NuxtHub-style KV adapter with SQL provider interop |
|
|
70
191
|
|
|
71
|
-
###
|
|
192
|
+
### Scenario Catalog
|
|
72
193
|
|
|
73
194
|
| Scenario Key | Scenario | What Happens | Workload Per Run |
|
|
74
195
|
| ----------------- | -------------------------------- | --------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------- |
|
|
@@ -77,71 +198,56 @@ CollegeDB includes a comprehensive benchmark suite covering real-world latency a
|
|
|
77
198
|
| migration_mapping | Migration-style mapping creation | Inserts legacy records on a fixed shard, then builds shard mappings in batch and validates routing. | 10 iterations; 20 legacy records mapped per iteration |
|
|
78
199
|
| bulk_crud | Bulk CRUD pressure | Performs bulk inserts, half updates, and full delete sweep, then validates shard-wide totals. | 7 iterations; 160 inserts + 80 updates + 160 deletes per iteration |
|
|
79
200
|
| indexing | Indexed query scan | Creates an index on posts(user_id) and repeatedly queries the indexed path. | 15 iterations after warmup dataset build |
|
|
201
|
+
| metadata_fetch | Metadata inspection | Reads table metadata/introspection rows from one shard. | 14 iterations; 1 metadata query per iteration |
|
|
202
|
+
| pragma_or_info | PRAGMA / server info | Runs provider-specific PRAGMA/info query to sample low-level metadata latency. | 14 iterations; 1 pragma/info query per iteration |
|
|
203
|
+
| counting | Cross-shard counting | Counts users across all shards to measure fanout aggregation overhead. | 14 iterations; all-shard count aggregation per iteration |
|
|
204
|
+
| shard_fanout | Shard fanout query | Runs query fanout to all shards and aggregates shard-level responses. | 14 iterations; 1 all-shards query per iteration |
|
|
205
|
+
| reassignment | Shard reassignment flow | Creates a record, reassigns it to another shard, and verifies routed reads still succeed. | 10 iterations; insert + reassignment + verification per iteration |
|
|
80
206
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
| Combination | Basic CRUD | Advanced Operations | Migration | Bulk CRUD | Indexing | Overall Avg |
|
|
84
|
-
| --------------- | ------------------- | ------------------- | ------------------- | --------------------- | -------------------- | ----------- |
|
|
85
|
-
| cloudflare | 13.14 ms / 16.50 ms | 4.43 ms / 9.65 ms | 27.68 ms / 30.69 ms | 156.30 ms / 163.76 ms | 67.17 ms / 106.63 ms | 28.40 ms |
|
|
86
|
-
| postgres+redis | 2.77 ms / 3.90 ms | 3.11 ms / 4.64 ms | 6.55 ms / 8.07 ms | 42.33 ms / 80.67 ms | 0.34 ms / 0.61 ms | 5.87 ms |
|
|
87
|
-
| postgres+valkey | 1.65 ms / 2.23 ms | 2.10 ms / 2.82 ms | 5.60 ms / 6.05 ms | 33.13 ms / 43.69 ms | 0.30 ms / 0.46 ms | 4.64 ms |
|
|
88
|
-
| mysql+redis | 5.11 ms / 8.38 ms | 5.45 ms / 8.51 ms | 27.41 ms / 61.56 ms | 92.70 ms / 139.70 ms | 0.49 ms / 1.22 ms | 13.91 ms |
|
|
89
|
-
| mysql+valkey | 4.99 ms / 6.66 ms | 4.21 ms / 6.42 ms | 21.68 ms / 27.20 ms | 87.67 ms / 109.44 ms | 0.55 ms / 1.92 ms | 12.54 ms |
|
|
90
|
-
| mariadb+redis | 2.64 ms / 5.90 ms | 3.02 ms / 7.55 ms | 6.48 ms / 7.66 ms | 46.99 ms / 58.08 ms | 0.37 ms / 1.08 ms | 6.29 ms |
|
|
91
|
-
| mariadb+valkey | 2.34 ms / 4.58 ms | 2.91 ms / 5.69 ms | 5.73 ms / 7.35 ms | 45.04 ms / 61.42 ms | 0.36 ms / 0.79 ms | 5.96 ms |
|
|
92
|
-
| sqlite+redis | 2.21 ms / 3.84 ms | 2.43 ms / 3.14 ms | 10.49 ms / 17.31 ms | 140.85 ms / 184.48 ms | 0.07 ms / 0.14 ms | 15.87 ms |
|
|
93
|
-
| sqlite+valkey | 1.36 ms / 1.80 ms | 2.06 ms / 2.70 ms | 6.36 ms / 8.77 ms | 121.13 ms / 156.31 ms | 0.06 ms / 0.14 ms | 13.42 ms |
|
|
94
|
-
|
|
95
|
-
### Overview
|
|
96
|
-
|
|
97
|
-
CollegeDB includes an integration benchmark suite covering both local provider matrices and Cloudflare Worker routing paths.
|
|
207
|
+
### Report Matrices
|
|
98
208
|
|
|
99
|
-
|
|
209
|
+
Each generated report includes:
|
|
100
210
|
|
|
101
|
-
- `
|
|
102
|
-
- `
|
|
103
|
-
- `
|
|
104
|
-
- `
|
|
105
|
-
- `
|
|
106
|
-
- `
|
|
107
|
-
- `pragma_or_info`: provider-specific pragma/info query latency
|
|
108
|
-
- `counting`: shard-wide aggregate counting
|
|
109
|
-
- `shard_fanout`: all-shards fanout query aggregation
|
|
110
|
-
- `reassignment`: shard reassignment and routed-read validation
|
|
211
|
+
- `Matrix: SQL x KV (Overall)`
|
|
212
|
+
- `Matrix: Adapter Profiles (Overall Avg)`
|
|
213
|
+
- `Matrix: Core Scenario Latency (avg/p95)`
|
|
214
|
+
- `Matrix: Introspection and Routing Latency (avg/p95)`
|
|
215
|
+
- `Cloudflare Worker (wrangler dev --local)`
|
|
216
|
+
- `Matrix: Cloudflare Adapter Profiles (Overall Avg)`
|
|
111
217
|
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
- an interpretation guide (`How To Read This Report`)
|
|
115
|
-
- a benchmark catalog with per-run workload details
|
|
116
|
-
- a compact overall matrix (passed/failed/skipped + overall average)
|
|
117
|
-
- split scenario matrices for core workload latency and introspection/routing latency
|
|
118
|
-
|
|
119
|
-
## Installation
|
|
218
|
+
### Common Commands
|
|
120
219
|
|
|
121
220
|
```bash
|
|
122
|
-
bun
|
|
123
|
-
|
|
124
|
-
|
|
221
|
+
bun run test:sandbox
|
|
222
|
+
bun run test:sandbox:drizzle
|
|
223
|
+
bun run test:sandbox:nuxthub
|
|
224
|
+
bun run test:sandbox:hyperdrive
|
|
125
225
|
```
|
|
126
226
|
|
|
227
|
+
For Docker-based benchmark details and filtering options, see [Sandbox Benchmarks (Docker Compose)](#sandbox-benchmarks-docker-compose).
|
|
228
|
+
|
|
127
229
|
## Provider Adapters
|
|
128
230
|
|
|
129
231
|
CollegeDB can run with either native Cloudflare bindings or custom providers as long as they match the exported `KVStorage` and `SQLDatabase` interfaces.
|
|
130
232
|
|
|
233
|
+
Drizzle interop is enabled by passing a Drizzle `sql` tag as the optional second argument to `createPostgreSQLProvider`, `createMySQLProvider`, or `createSQLiteProvider`.
|
|
234
|
+
|
|
131
235
|
Supported adapters:
|
|
132
236
|
|
|
133
237
|
- `createRedisKVProvider`
|
|
134
238
|
- `createValkeyKVProvider`
|
|
135
|
-
- `
|
|
136
|
-
- `
|
|
137
|
-
- `
|
|
239
|
+
- `createNuxtHubKVProvider`
|
|
240
|
+
- `createPostgreSQLProvider`
|
|
241
|
+
- `createMySQLProvider`
|
|
242
|
+
- `createSQLiteProvider`
|
|
243
|
+
- `createDrizzleSQLProvider` (compatibility helper)
|
|
138
244
|
- `createHyperdrivePostgresProvider`
|
|
139
245
|
- `createHyperdriveMySQLProvider`
|
|
140
246
|
|
|
141
247
|
```typescript
|
|
142
248
|
import { createClient as createRedisClient } from 'redis';
|
|
143
249
|
import { Pool } from 'pg';
|
|
144
|
-
import {
|
|
250
|
+
import { createPostgreSQLProvider, createRedisKVProvider, initialize, run, type CollegeDBConfig } from '@earth-app/collegedb';
|
|
145
251
|
|
|
146
252
|
const redisClient = createRedisClient({ url: process.env.REDIS_URL });
|
|
147
253
|
const pgPool = new Pool({ connectionString: process.env.POSTGRES_URL });
|
|
@@ -149,7 +255,7 @@ const pgPool = new Pool({ connectionString: process.env.POSTGRES_URL });
|
|
|
149
255
|
const config: CollegeDBConfig = {
|
|
150
256
|
kv: createRedisKVProvider(redisClient),
|
|
151
257
|
shards: {
|
|
152
|
-
'pg-east':
|
|
258
|
+
'pg-east': createPostgreSQLProvider(pgPool)
|
|
153
259
|
},
|
|
154
260
|
strategy: 'hash',
|
|
155
261
|
disableAutoMigration: true
|
|
@@ -166,8 +272,101 @@ bootstrap().catch(console.error);
|
|
|
166
272
|
|
|
167
273
|
For Hyperdrive-backed SQL connections, use `createHyperdrivePostgresProvider` or `createHyperdriveMySQLProvider` with your database client factory.
|
|
168
274
|
|
|
275
|
+
### NuxtHub Runtime Example (D1 + KV)
|
|
276
|
+
|
|
277
|
+
```typescript
|
|
278
|
+
import { db } from '@nuxthub/db';
|
|
279
|
+
import { kv } from '@nuxthub/kv';
|
|
280
|
+
import { sql } from 'drizzle-orm';
|
|
281
|
+
import { createNuxtHubKVProvider, createSQLiteProvider, initialize, run, first } from '@earth-app/collegedb';
|
|
282
|
+
|
|
283
|
+
initialize({
|
|
284
|
+
kv: createNuxtHubKVProvider(kv),
|
|
285
|
+
shards: {
|
|
286
|
+
'db-primary': createSQLiteProvider(db, sql)
|
|
287
|
+
},
|
|
288
|
+
strategy: 'hash'
|
|
289
|
+
});
|
|
290
|
+
|
|
291
|
+
await run('draft:home', 'INSERT OR REPLACE INTO drafts (id, content) VALUES (?, ?)', ['draft:home', '# Home']);
|
|
292
|
+
|
|
293
|
+
const draft = await first<{ id: string; content: string }>('draft:home', 'SELECT id, content FROM drafts WHERE id = ?', ['draft:home']);
|
|
294
|
+
```
|
|
295
|
+
|
|
296
|
+
### Keep Drizzle Schema + Migrations, Route Queries with CollegeDB
|
|
297
|
+
|
|
298
|
+
CollegeDB does not replace your Drizzle schema or NuxtHub migration workflow.
|
|
299
|
+
|
|
300
|
+
```bash
|
|
301
|
+
npx nuxt db generate
|
|
302
|
+
npx nuxt db migrate
|
|
303
|
+
```
|
|
304
|
+
|
|
305
|
+
Use those migrations as-is, then route runtime reads/writes through CollegeDB adapters.
|
|
306
|
+
|
|
169
307
|
For a complete non-Cloudflare setup, see `examples/provider-sandbox.ts`.
|
|
170
308
|
|
|
309
|
+
## NuxtHub + Drizzle Recipes
|
|
310
|
+
|
|
311
|
+
### Multi-Vendor Shards (Cloudflare + Postgres/MySQL)
|
|
312
|
+
|
|
313
|
+
NuxtHub supports multiple deployment/database vendors. CollegeDB can shard across any SQL backends that Drizzle can connect to.
|
|
314
|
+
|
|
315
|
+
```typescript
|
|
316
|
+
import { sql } from 'drizzle-orm';
|
|
317
|
+
import { drizzle as drizzlePg } from 'drizzle-orm/postgres-js';
|
|
318
|
+
import { drizzle as drizzleMySQL } from 'drizzle-orm/mysql2';
|
|
319
|
+
import { drizzle as drizzleD1 } from 'drizzle-orm/d1';
|
|
320
|
+
import { kv } from '@nuxthub/kv';
|
|
321
|
+
import postgres from 'postgres';
|
|
322
|
+
import mysql from 'mysql2/promise';
|
|
323
|
+
import {
|
|
324
|
+
createMySQLProvider,
|
|
325
|
+
createNuxtHubKVProvider,
|
|
326
|
+
createPostgreSQLProvider,
|
|
327
|
+
createSQLiteProvider,
|
|
328
|
+
initialize,
|
|
329
|
+
run
|
|
330
|
+
} from '@earth-app/collegedb';
|
|
331
|
+
|
|
332
|
+
const pgClient = postgres(process.env.POSTGRES_URL!);
|
|
333
|
+
const mysqlPool = mysql.createPool(process.env.MYSQL_URL!);
|
|
334
|
+
|
|
335
|
+
function setup(env: { DB_CF: D1Database }) {
|
|
336
|
+
initialize({
|
|
337
|
+
kv: createNuxtHubKVProvider(kv),
|
|
338
|
+
shards: {
|
|
339
|
+
'db-cf': createSQLiteProvider(drizzleD1(env.DB_CF), sql),
|
|
340
|
+
'db-pg': createPostgreSQLProvider(drizzlePg(pgClient), sql),
|
|
341
|
+
'db-mysql': createMySQLProvider(drizzleMySQL(mysqlPool), sql)
|
|
342
|
+
},
|
|
343
|
+
strategy: 'hash'
|
|
344
|
+
});
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
export default defineEventHandler(async (event) => {
|
|
348
|
+
setup(event.context.cloudflare.env);
|
|
349
|
+
|
|
350
|
+
await run('tenant:acme:user:1', 'INSERT INTO users (id, name) VALUES (?, ?)', ['tenant:acme:user:1', 'Ada']);
|
|
351
|
+
});
|
|
352
|
+
```
|
|
353
|
+
|
|
354
|
+
### Keep NuxtHub Cache/KV Semantics Intact
|
|
355
|
+
|
|
356
|
+
Use NuxtHub KV for app cache while CollegeDB uses its own key namespace for shard mappings:
|
|
357
|
+
|
|
358
|
+
```typescript
|
|
359
|
+
import { kv } from '@nuxthub/kv';
|
|
360
|
+
import { first } from '@earth-app/collegedb';
|
|
361
|
+
|
|
362
|
+
const cacheKey = `nuxtpress:post:${slug}`;
|
|
363
|
+
const cached = await kv.get(cacheKey);
|
|
364
|
+
if (cached) return cached;
|
|
365
|
+
|
|
366
|
+
const post = await first(cacheKey, 'SELECT * FROM blog_posts WHERE slug = ? LIMIT 1', [slug]);
|
|
367
|
+
await kv.set(cacheKey, post, { ttl: 3600 });
|
|
368
|
+
```
|
|
369
|
+
|
|
171
370
|
## Sandbox Benchmarks (Docker Compose)
|
|
172
371
|
|
|
173
372
|
CollegeDB ships with an integration sandbox runner that benchmarks real latency across provider combinations.
|
|
@@ -199,16 +398,21 @@ bun run test:sandbox:cloudflare
|
|
|
199
398
|
Provider filters:
|
|
200
399
|
|
|
201
400
|
```bash
|
|
202
|
-
# One SQL provider against
|
|
401
|
+
# One SQL provider against all KV providers (native profile by default)
|
|
203
402
|
bun run test:sandbox:mysql
|
|
204
403
|
bun run test:sandbox:postgres
|
|
205
404
|
bun run test:sandbox:mariadb
|
|
206
405
|
bun run test:sandbox:sqlite
|
|
207
406
|
|
|
208
|
-
# One KV provider against all SQL providers
|
|
407
|
+
# One KV provider against all SQL providers (native profile by default)
|
|
209
408
|
bun run test:sandbox:redis
|
|
210
409
|
bun run test:sandbox:valkey
|
|
211
410
|
|
|
411
|
+
# Run all SQL x KV combinations for one adapter profile
|
|
412
|
+
bun run test:sandbox:drizzle
|
|
413
|
+
bun run test:sandbox:nuxthub
|
|
414
|
+
bun run test:sandbox:hyperdrive
|
|
415
|
+
|
|
212
416
|
# Explicit pairwise combinations
|
|
213
417
|
bun run test:sandbox:postgres+redis
|
|
214
418
|
bun run test:sandbox:postgres+valkey
|
|
@@ -225,7 +429,7 @@ Output behavior:
|
|
|
225
429
|
- Every run writes a timestamped Markdown report to `sandbox/results/`
|
|
226
430
|
- `sandbox/results/latest.md` is always updated to the newest report
|
|
227
431
|
- The runner prints the report in-terminal using Bun's Markdown renderer with ANSI formatting
|
|
228
|
-
- `test:sandbox`
|
|
432
|
+
- `test:sandbox` includes native, drizzle, hyperdrive, and nuxthub adapter profiles across supported SQL/KV combinations plus Cloudflare profile runs
|
|
229
433
|
|
|
230
434
|
Benchmark coverage includes:
|
|
231
435
|
|
|
@@ -704,7 +908,7 @@ for (const [table, pkColumn] of Object.entries(customIntegration)) {
|
|
|
704
908
|
}
|
|
705
909
|
```
|
|
706
910
|
|
|
707
|
-
##
|
|
911
|
+
## API Reference
|
|
708
912
|
|
|
709
913
|
| Function | Description | Parameters |
|
|
710
914
|
| ------------------------------------------ | ---------------------------------------------------------------- | -------------------------- |
|
|
@@ -733,9 +937,11 @@ for (const [table, pkColumn] of Object.entries(customIntegration)) {
|
|
|
733
937
|
| ---------------------------------------------------------- | -------------------------------------------------------------------- | -------------------------------------------------------- |
|
|
734
938
|
| `createRedisKVProvider(client, options?)` | Adapt a Redis client to CollegeDB's `KVStorage` contract | `RedisLikeClient, { scanCount?: number }` |
|
|
735
939
|
| `createValkeyKVProvider(client, options?)` | Adapt a Valkey client to CollegeDB's `KVStorage` contract | `RedisLikeClient, { scanCount?: number }` |
|
|
736
|
-
| `
|
|
737
|
-
| `
|
|
738
|
-
| `
|
|
940
|
+
| `createNuxtHubKVProvider(client)` | Adapt NuxtHub/Unstorage-style KV clients to `KVStorage` | `NuxtHubKVLike` |
|
|
941
|
+
| `createPostgreSQLProvider(client, sqlTag?)` | Adapt PostgreSQL or Drizzle PostgreSQL clients | `PostgresClientLike, sqlTag?` |
|
|
942
|
+
| `createMySQLProvider(client, sqlTag?)` | Adapt MySQL/MariaDB or Drizzle MySQL/MariaDB clients | `MySQLClientLike, sqlTag?` |
|
|
943
|
+
| `createSQLiteProvider(client, sqlTag?)` | Adapt SQLite/D1 or Drizzle SQLite/D1 clients | `SQLiteClientLike, sqlTag?` |
|
|
944
|
+
| `createDrizzleSQLProvider(client, sqlTag)` | Generic Drizzle adapter (optional helper) | `DrizzleClientLike, DrizzleSqlTagLike` |
|
|
739
945
|
| `createHyperdrivePostgresProvider(binding, clientFactory)` | Create a PostgreSQL `SQLDatabase` adapter using a Hyperdrive binding | `HyperdriveBindingLike, HyperdrivePostgresClientFactory` |
|
|
740
946
|
| `createHyperdriveMySQLProvider(binding, clientFactory)` | Create a MySQL `SQLDatabase` adapter using a Hyperdrive binding | `HyperdriveBindingLike, HyperdriveMySQLClientFactory` |
|
|
741
947
|
| `isKVStorage(value)` | Runtime guard for `KVStorage` | `unknown` |
|
|
@@ -990,7 +1196,10 @@ CollegeDB exports TypeScript types for better development experience and type sa
|
|
|
990
1196
|
| ----------------------- | ------------------------------ | --------------------------------------------------- |
|
|
991
1197
|
| `CollegeDBConfig` | Main configuration object | `{ kv, shards, strategy }` |
|
|
992
1198
|
| `KVStorage` | Provider-agnostic KV contract | `createRedisKVProvider(redisClient)` |
|
|
993
|
-
| `SQLDatabase` | Provider-agnostic SQL contract | `
|
|
1199
|
+
| `SQLDatabase` | Provider-agnostic SQL contract | `createPostgreSQLProvider(pgPool)` |
|
|
1200
|
+
| `NuxtHubKVLike` | NuxtHub/Unstorage KV contract | `createNuxtHubKVProvider(kv)` |
|
|
1201
|
+
| `DrizzleClientLike` | Minimal Drizzle DB contract | `createPostgreSQLProvider(drizzleDb, sql)` |
|
|
1202
|
+
| `DrizzleSqlTagLike` | Drizzle SQL tag contract | `createSQLiteProvider(drizzleDb, sql)` |
|
|
994
1203
|
| `QueryResult` | Standard query response shape | `{ success, results, meta }` |
|
|
995
1204
|
| `QueryResultMeta` | Query execution metadata | `{ duration, changes?, last_row_id? }` |
|
|
996
1205
|
| `ShardingStrategy` | Single strategy options | `'hash' \| 'location' \| 'round-robin' \| 'random'` |
|
|
@@ -1026,7 +1235,7 @@ const config: CollegeDBConfig = {
|
|
|
1026
1235
|
};
|
|
1027
1236
|
```
|
|
1028
1237
|
|
|
1029
|
-
##
|
|
1238
|
+
## Architecture
|
|
1030
1239
|
|
|
1031
1240
|
```txt
|
|
1032
1241
|
┌─────────────────────────────────────────────────────────────┐
|
|
@@ -1097,7 +1306,7 @@ const config: CollegeDBConfig = {
|
|
|
1097
1306
|
- **Random**: Random shard selection for load balancing
|
|
1098
1307
|
- **Location**: Geographic proximity-based allocation for optimal latency
|
|
1099
1308
|
|
|
1100
|
-
##
|
|
1309
|
+
## Cloudflare Setup
|
|
1101
1310
|
|
|
1102
1311
|
### 1. Create D1 Databases
|
|
1103
1312
|
|
|
@@ -1301,7 +1510,7 @@ wrangler deploy
|
|
|
1301
1510
|
wrangler deploy --env production
|
|
1302
1511
|
```
|
|
1303
1512
|
|
|
1304
|
-
##
|
|
1513
|
+
## Monitoring and Maintenance
|
|
1305
1514
|
|
|
1306
1515
|
### Shard Statistics
|
|
1307
1516
|
|
|
@@ -1585,7 +1794,7 @@ export default {
|
|
|
1585
1794
|
};
|
|
1586
1795
|
```
|
|
1587
1796
|
|
|
1588
|
-
##
|
|
1797
|
+
## Performance Analysis
|
|
1589
1798
|
|
|
1590
1799
|
### Scaling Performance Comparison
|
|
1591
1800
|
|
|
@@ -1975,7 +2184,7 @@ initialize({
|
|
|
1975
2184
|
- Cost-sensitive deployments at small scale
|
|
1976
2185
|
- **Single-strategy applications** where reads and writes have identical performance needs
|
|
1977
2186
|
|
|
1978
|
-
##
|
|
2187
|
+
## Advanced Configuration
|
|
1979
2188
|
|
|
1980
2189
|
### Custom Allocation Strategy
|
|
1981
2190
|
|
|
@@ -2292,7 +2501,7 @@ async function allocateWithFallback(coordinator: DurableObjectNamespace, primary
|
|
|
2292
2501
|
}
|
|
2293
2502
|
```
|
|
2294
2503
|
|
|
2295
|
-
##
|
|
2504
|
+
## Quick Reference
|
|
2296
2505
|
|
|
2297
2506
|
### Strategy Selection Guide
|
|
2298
2507
|
|
|
@@ -2410,7 +2619,7 @@ async function allocateWithFallback(coordinator: DurableObjectNamespace, primary
|
|
|
2410
2619
|
| `me` | Middle East | Dubai |
|
|
2411
2620
|
| `af` | Africa | Johannesburg |
|
|
2412
2621
|
|
|
2413
|
-
##
|
|
2622
|
+
## Contributing
|
|
2414
2623
|
|
|
2415
2624
|
1. Fork the repository
|
|
2416
2625
|
2. Create a feature branch: `git checkout -b feature/amazing-feature`
|
|
@@ -2418,18 +2627,18 @@ async function allocateWithFallback(coordinator: DurableObjectNamespace, primary
|
|
|
2418
2627
|
4. Push to branch: `git push origin feature/amazing-feature`
|
|
2419
2628
|
5. Submit a pull request
|
|
2420
2629
|
|
|
2421
|
-
##
|
|
2630
|
+
## License
|
|
2422
2631
|
|
|
2423
2632
|
This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
|
|
2424
2633
|
|
|
2425
|
-
##
|
|
2634
|
+
## Links
|
|
2426
2635
|
|
|
2427
2636
|
- [Cloudflare D1 Documentation](https://developers.cloudflare.com/d1/)
|
|
2428
2637
|
- [Cloudflare KV Documentation](https://developers.cloudflare.com/kv/)
|
|
2429
2638
|
- [Cloudflare Workers Documentation](https://developers.cloudflare.com/workers/)
|
|
2430
2639
|
- [Durable Objects Documentation](https://developers.cloudflare.com/durable-objects/)
|
|
2431
2640
|
|
|
2432
|
-
##
|
|
2641
|
+
## Support
|
|
2433
2642
|
|
|
2434
2643
|
- 📖 [Documentation](https://earth-app.github.io/CollegeDB)
|
|
2435
2644
|
- 🐛 [Report Issues](https://github.com/earth-app/CollegeDB/issues)
|
package/dist/errors.d.ts
CHANGED
package/dist/index.d.ts
CHANGED
|
@@ -8,11 +8,11 @@
|
|
|
8
8
|
* @author Gregory Mitchell
|
|
9
9
|
* @license MIT
|
|
10
10
|
*/
|
|
11
|
-
export { all, allAllShards, allShard, collegedb, createSchema, first, firstAllShards, firstShard, flush, getClosestRegionFromIP, getDatabaseSizeForShard, getShardStats, initialize, initializeAsync, listKnownShards, prepare, reassignShard, resetConfig, run, runAllShards, runShard } from './router
|
|
12
|
-
export { ShardCoordinator } from './durable
|
|
13
|
-
export { CollegeDBError } from './errors
|
|
14
|
-
export { KVShardMapper } from './kvmap
|
|
15
|
-
export { createHyperdriveMySQLProvider, createHyperdrivePostgresProvider, createMySQLProvider
|
|
16
|
-
export { autoDetectAndMigrate, checkMigrationNeeded, clearMigrationCache, clearShardMigrationCache, createMappingsForExistingKeys, createSchemaAcrossShards, discoverExistingPrimaryKeys, discoverExistingRecordsWithColumns, dropSchema, integrateExistingDatabase, listTables, migrateRecord, schemaExists, validateTableForSharding, type IntegrationOptions, type IntegrationResult, type ValidationResult } from './migrations
|
|
17
|
-
export type { CollegeDBConfig, D1Region, Env, KVListResult, KVStorage, MixedShardingStrategy, OperationType, PreparedStatement, QueryResult, QueryResultMeta, SQLDatabase, ShardCoordinatorState, ShardLocation, ShardMapping, ShardStats, ShardingStrategy } from './types
|
|
11
|
+
export { all, allAllShards, allShard, collegedb, createSchema, first, firstAllShards, firstShard, flush, getClosestRegionFromIP, getDatabaseSizeForShard, getShardStats, initialize, initializeAsync, listKnownShards, prepare, reassignShard, resetConfig, run, runAllShards, runShard } from './router';
|
|
12
|
+
export { ShardCoordinator } from './durable';
|
|
13
|
+
export { CollegeDBError } from './errors';
|
|
14
|
+
export { KVShardMapper } from './kvmap';
|
|
15
|
+
export { createDrizzleSQLProvider, createHyperdriveMySQLProvider, createHyperdrivePostgresProvider, createMySQLProvider, createNuxtHubKVProvider, createPostgreSQLProvider, createRedisKVProvider, createSQLiteProvider, createValkeyKVProvider, isKVStorage, isSQLDatabase, type DrizzleClientLike, type DrizzleSqlChunkLike, type DrizzleSqlTagLike, type HyperdriveBindingLike, type HyperdriveMySQLClientFactory, type HyperdrivePostgresClientFactory, type MySQLClientLike, type NuxtHubKVLike, type PostgresClientLike, type RedisLikeClient, type SQLiteClientLike } from './providers';
|
|
16
|
+
export { autoDetectAndMigrate, checkMigrationNeeded, clearMigrationCache, clearShardMigrationCache, createMappingsForExistingKeys, createSchemaAcrossShards, discoverExistingPrimaryKeys, discoverExistingRecordsWithColumns, dropSchema, integrateExistingDatabase, listTables, migrateRecord, schemaExists, validateTableForSharding, type IntegrationOptions, type IntegrationResult, type ValidationResult } from './migrations';
|
|
17
|
+
export type { CollegeDBConfig, D1Region, Env, KVListResult, KVStorage, MixedShardingStrategy, OperationType, PreparedStatement, QueryResult, QueryResultMeta, SQLDatabase, ShardCoordinatorState, ShardLocation, ShardMapping, ShardStats, ShardingStrategy } from './types';
|
|
18
18
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,EACN,GAAG,EACH,YAAY,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,KAAK,EACL,cAAc,EACd,UAAU,EACV,KAAK,EACL,sBAAsB,EACtB,uBAAuB,EACvB,aAAa,EACb,UAAU,EACV,eAAe,EACf,eAAe,EACf,OAAO,EACP,aAAa,EACb,WAAW,EACX,GAAG,EACH,YAAY,EACZ,QAAQ,EACR,MAAM,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,EACN,GAAG,EACH,YAAY,EACZ,QAAQ,EACR,SAAS,EACT,YAAY,EACZ,KAAK,EACL,cAAc,EACd,UAAU,EACV,KAAK,EACL,sBAAsB,EACtB,uBAAuB,EACvB,aAAa,EACb,UAAU,EACV,eAAe,EACf,eAAe,EACf,OAAO,EACP,aAAa,EACb,WAAW,EACX,GAAG,EACH,YAAY,EACZ,QAAQ,EACR,MAAM,UAAU,CAAC;AAGlB,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAGxC,OAAO,EACN,wBAAwB,EACxB,6BAA6B,EAC7B,gCAAgC,EAChC,mBAAmB,EACnB,uBAAuB,EACvB,wBAAwB,EACxB,qBAAqB,EACrB,oBAAoB,EACpB,sBAAsB,EACtB,WAAW,EACX,aAAa,EACb,KAAK,iBAAiB,EACtB,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,EACtB,KAAK,qBAAqB,EAC1B,KAAK,4BAA4B,EACjC,KAAK,+BAA+B,EACpC,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,kBAAkB,EACvB,KAAK,eAAe,EACpB,KAAK,gBAAgB,EACrB,MAAM,aAAa,CAAC;AAGrB,OAAO,EACN,oBAAoB,EACpB,oBAAoB,EACpB,mBAAmB,EACnB,wBAAwB,EACxB,6BAA6B,EAC7B,wBAAwB,EACxB,2BAA2B,EAC3B,kCAAkC,EAClC,UAAU,EACV,yBAAyB,EACzB,UAAU,EACV,aAAa,EACb,YAAY,EACZ,wBAAwB,EACxB,KAAK,kBAAkB,EACvB,KAAK,iBAAiB,EACtB,KAAK,gBAAgB,EACrB,MAAM,cAAc,CAAC;AAGtB,YAAY,EACX,eAAe,EACf,QAAQ,EACR,GAAG,EACH,YAAY,EACZ,SAAS,EACT,qBAAqB,EACrB,aAAa,EACb,iBAAiB,EACjB,WAAW,EACX,eAAe,EACf,WAAW,EACX,qBAAqB,EACrB,aAAa,EACb,YAAY,EACZ,UAAU,EACV,gBAAgB,EAChB,MAAM,SAAS,CAAC"}
|