@devxcrew/framework 0.1.10 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/modules/database/common/database.schema.d.ts +44 -0
- package/dist/modules/database/common/database.schema.js +49 -0
- package/dist/modules/database/common/database.types.d.ts +26 -0
- package/dist/modules/database/common/database.types.js +1 -0
- package/dist/modules/database/common/index.d.ts +2 -0
- package/dist/modules/database/common/index.js +1 -0
- package/dist/modules/database/database.connections.d.ts +45 -0
- package/dist/modules/database/database.connections.js +196 -0
- package/dist/modules/database/database.controller.d.ts +1 -0
- package/dist/modules/database/database.controller.js +1 -0
- package/dist/modules/database/database.execution.d.ts +30 -0
- package/dist/modules/database/database.execution.js +109 -0
- package/dist/modules/database/database.migration.d.ts +11 -0
- package/dist/modules/database/database.migration.js +80 -0
- package/dist/modules/database/database.provider.d.ts +38 -0
- package/dist/modules/database/database.provider.js +39 -0
- package/dist/modules/database/database.repository.d.ts +1 -0
- package/dist/modules/database/database.repository.js +1 -0
- package/dist/modules/database/database.routes.d.ts +1 -0
- package/dist/modules/database/database.routes.js +1 -0
- package/dist/modules/database/database.schema.d.ts +1 -0
- package/dist/modules/database/database.schema.js +1 -0
- package/dist/modules/database/database.seed.d.ts +3 -0
- package/dist/modules/database/database.seed.js +7 -0
- package/dist/modules/database/database.service.d.ts +7 -0
- package/dist/modules/database/database.service.js +24 -0
- package/dist/modules/database/database.transfer.d.ts +17 -0
- package/dist/modules/database/database.transfer.js +79 -0
- package/dist/modules/database/database.types.d.ts +1 -0
- package/dist/modules/database/database.types.js +1 -0
- package/dist/modules/database/drivers/mariadb/index.d.ts +5 -0
- package/dist/modules/database/drivers/mariadb/index.js +5 -0
- package/dist/modules/database/drivers/mariadb/mariadb.backup.d.ts +3 -0
- package/dist/modules/database/drivers/mariadb/mariadb.backup.js +141 -0
- package/dist/modules/database/drivers/mariadb/mariadb.compatibility.d.ts +6 -0
- package/dist/modules/database/drivers/mariadb/mariadb.compatibility.js +88 -0
- package/dist/modules/database/drivers/mariadb/mariadb.connection.d.ts +3 -0
- package/dist/modules/database/drivers/mariadb/mariadb.connection.js +22 -0
- package/dist/modules/database/drivers/mariadb/mariadb.schema.d.ts +14 -0
- package/dist/modules/database/drivers/mariadb/mariadb.schema.js +29 -0
- package/dist/modules/database/drivers/sqlite/index.d.ts +3 -0
- package/dist/modules/database/drivers/sqlite/index.js +3 -0
- package/dist/modules/database/drivers/sqlite/sqlite.backup.d.ts +2 -0
- package/dist/modules/database/drivers/sqlite/sqlite.backup.js +45 -0
- package/dist/modules/database/drivers/sqlite/sqlite.connection.d.ts +13 -0
- package/dist/modules/database/drivers/sqlite/sqlite.connection.js +41 -0
- package/dist/modules/database/drivers/sqlite/sqlite.schema.d.ts +3 -0
- package/dist/modules/database/drivers/sqlite/sqlite.schema.js +9 -0
- package/dist/modules/database/index.d.ts +12 -0
- package/dist/modules/database/index.js +11 -0
- package/dist/modules/database/operations/database.backup.d.ts +3 -0
- package/dist/modules/database/operations/database.backup.js +24 -0
- package/dist/modules/database/operations/sqlite-import.d.ts +8 -0
- package/dist/modules/database/operations/sqlite-import.js +96 -0
- package/dist/modules/settings/index.d.ts +2 -0
- package/dist/modules/settings/index.js +1 -0
- package/dist/modules/settings/settings.controller.d.ts +1 -0
- package/dist/modules/settings/settings.controller.js +1 -0
- package/dist/modules/settings/settings.migration.d.ts +1 -0
- package/dist/modules/settings/settings.migration.js +1 -0
- package/dist/modules/settings/settings.provider.d.ts +2 -0
- package/dist/modules/settings/settings.provider.js +9 -0
- package/dist/modules/settings/settings.repository.d.ts +1 -0
- package/dist/modules/settings/settings.repository.js +1 -0
- package/dist/modules/settings/settings.routes.d.ts +1 -0
- package/dist/modules/settings/settings.routes.js +1 -0
- package/dist/modules/settings/settings.schema.d.ts +1 -0
- package/dist/modules/settings/settings.schema.js +1 -0
- package/dist/modules/settings/settings.seed.d.ts +1 -0
- package/dist/modules/settings/settings.seed.js +1 -0
- package/dist/modules/settings/settings.service.d.ts +13 -0
- package/dist/modules/settings/settings.service.js +93 -0
- package/dist/modules/settings/settings.types.d.ts +11 -0
- package/dist/modules/settings/settings.types.js +1 -0
- package/dist/runtime/config.js +3 -2
- package/package.json +5 -2
package/README.md
CHANGED
|
@@ -30,7 +30,7 @@ const statusToken = createModuleToken<{ routes: ApiRoute[] }>("status");
|
|
|
30
30
|
const status = defineModuleProvider({
|
|
31
31
|
token: statusToken,
|
|
32
32
|
dependencies: [],
|
|
33
|
-
create: () => ({
|
|
33
|
+
create: (): { routes: ApiRoute[] } => ({
|
|
34
34
|
routes: [
|
|
35
35
|
{
|
|
36
36
|
method: "GET",
|
|
@@ -74,6 +74,8 @@ server.listen(3000);
|
|
|
74
74
|
|
|
75
75
|
## Operations
|
|
76
76
|
|
|
77
|
+
- `readApplicationConfig` validates `APP_NAME`, `APP_PORT`, `APP_MODE`, and the public `APP_URL`. In development, the URL port must match `APP_PORT`. In production, a reverse proxy can expose a different HTTPS port. Set `APP_HOST` to the listen address when it differs from the public host.
|
|
78
|
+
- Apps load `.env` with their own startup tool and validate their extra settings. Pass those values to `readApplicationConfig`. The Framework does not read files or secrets on import.
|
|
77
79
|
- `createApplicationServer` sets basic security headers. Configure exact CORS origins, trusted proxy addresses, and rate limits for each app.
|
|
78
80
|
- The built-in rate limiter is local to one process. Set `rateLimit.store` to an app-owned atomic shared store when deployment uses multiple instances. Store failures return 503.
|
|
79
81
|
- Set `logger` for request logs. Set `onRequestComplete` to pass request data to a metrics or tracing adapter. Neither receives URL query values.
|
|
@@ -90,3 +92,9 @@ npm run release:check
|
|
|
90
92
|
```
|
|
91
93
|
|
|
92
94
|
Keep secrets in ignored environment files. Commit and publish through the repository release workflow only when authorized.
|
|
95
|
+
|
|
96
|
+
## Database and environment settings
|
|
97
|
+
|
|
98
|
+
Framework now owns createDatabaseProvider, DatabaseExecution, ResumableTransfer and createSettingsProvider.
|
|
99
|
+
See agent/DATABASE.md for ownership and development use.
|
|
100
|
+
These contracts are local source changes. Publish a new package version before using registry pins in other apps.
|
package/dist/index.d.ts
CHANGED
|
@@ -6,3 +6,5 @@ export { createValidationProvider, parseWithSchema, type ValidationProvider, } f
|
|
|
6
6
|
export { createLogger, type Logger, type LoggerOptions, type LogFields, type LogLevel, type LogRecord, } from "./modules/logger/logger.provider.js";
|
|
7
7
|
export { createHttpSecurityProvider, type HttpSecurityOptions, type HttpSecurityProvider, type RateLimitOptions, type RateLimitStore, } from "./modules/http/http-security.provider.js";
|
|
8
8
|
export { createHealthProvider, createAsyncHealthProvider, type HealthCheck, type HealthProvider, type HealthSnapshot, type AsyncHealthCheck, type AsyncHealthProvider, } from "./modules/health/health.provider.js";
|
|
9
|
+
export * from "./modules/database/index.js";
|
|
10
|
+
export * from "./modules/settings/index.js";
|
package/dist/index.js
CHANGED
|
@@ -6,3 +6,5 @@ export { createValidationProvider, parseWithSchema, } from "./modules/validation
|
|
|
6
6
|
export { createLogger, } from "./modules/logger/logger.provider.js";
|
|
7
7
|
export { createHttpSecurityProvider, } from "./modules/http/http-security.provider.js";
|
|
8
8
|
export { createHealthProvider, createAsyncHealthProvider, } from "./modules/health/health.provider.js";
|
|
9
|
+
export * from "./modules/database/index.js";
|
|
10
|
+
export * from "./modules/settings/index.js";
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
export declare function readDatabaseConfiguration(environment?: NodeJS.ProcessEnv): {
|
|
3
|
+
readonly driver: "mariadb";
|
|
4
|
+
readonly options: {
|
|
5
|
+
host: string;
|
|
6
|
+
port: number;
|
|
7
|
+
user: string;
|
|
8
|
+
password: string;
|
|
9
|
+
database: string;
|
|
10
|
+
connectionLimit: number;
|
|
11
|
+
connectTimeout: number;
|
|
12
|
+
charset: string;
|
|
13
|
+
supportBigNumbers: boolean;
|
|
14
|
+
bigNumberStrings: boolean;
|
|
15
|
+
multipleStatements: boolean;
|
|
16
|
+
timezone: string;
|
|
17
|
+
};
|
|
18
|
+
} | {
|
|
19
|
+
readonly path: string;
|
|
20
|
+
readonly driver: "sqlite";
|
|
21
|
+
readonly options?: undefined;
|
|
22
|
+
};
|
|
23
|
+
export declare const databaseCommandSchema: z.ZodEnum<{
|
|
24
|
+
check: "check";
|
|
25
|
+
connections: "connections";
|
|
26
|
+
migrate: "migrate";
|
|
27
|
+
seed: "seed";
|
|
28
|
+
"seed:tenancy": "seed:tenancy";
|
|
29
|
+
smoke: "smoke";
|
|
30
|
+
backup: "backup";
|
|
31
|
+
"verify-backup": "verify-backup";
|
|
32
|
+
"verify-restore": "verify-restore";
|
|
33
|
+
"import-sqlite": "import-sqlite";
|
|
34
|
+
}>;
|
|
35
|
+
export declare const connectionTargetSchema: z.ZodObject<{
|
|
36
|
+
key: z.ZodString;
|
|
37
|
+
driver: z.ZodEnum<{
|
|
38
|
+
mariadb: "mariadb";
|
|
39
|
+
sqlite: "sqlite";
|
|
40
|
+
}>;
|
|
41
|
+
databaseName: z.ZodNullable<z.ZodString>;
|
|
42
|
+
sqlitePath: z.ZodNullable<z.ZodString>;
|
|
43
|
+
version: z.ZodNumber;
|
|
44
|
+
}, z.core.$strict>;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { mariaDbConfig } from "../drivers/mariadb/index.js";
|
|
3
|
+
import { sqliteConfig } from "../drivers/sqlite/index.js";
|
|
4
|
+
export function readDatabaseConfiguration(environment = process.env) {
|
|
5
|
+
const normalized = { ...environment };
|
|
6
|
+
const driver = normalized.DB_DRIVER;
|
|
7
|
+
if (driver === "mariadb")
|
|
8
|
+
return { driver, options: mariaDbConfig(normalized) };
|
|
9
|
+
if (driver !== "sqlite")
|
|
10
|
+
throw new Error("DB_DRIVER must be mariadb or sqlite.");
|
|
11
|
+
return { driver, ...sqliteConfig(normalized) };
|
|
12
|
+
}
|
|
13
|
+
export const databaseCommandSchema = z.enum([
|
|
14
|
+
"connections",
|
|
15
|
+
"migrate",
|
|
16
|
+
"seed",
|
|
17
|
+
"seed:tenancy",
|
|
18
|
+
"check",
|
|
19
|
+
"smoke",
|
|
20
|
+
"backup",
|
|
21
|
+
"verify-backup",
|
|
22
|
+
"verify-restore",
|
|
23
|
+
"import-sqlite",
|
|
24
|
+
]);
|
|
25
|
+
export const connectionTargetSchema = z
|
|
26
|
+
.strictObject({
|
|
27
|
+
key: z.string().min(1).max(255),
|
|
28
|
+
driver: z.enum(["sqlite", "mariadb"]),
|
|
29
|
+
databaseName: z
|
|
30
|
+
.string()
|
|
31
|
+
.regex(/^[a-zA-Z][a-zA-Z0-9_]{0,63}$/)
|
|
32
|
+
.nullable(),
|
|
33
|
+
sqlitePath: z.string().trim().min(1).nullable(),
|
|
34
|
+
version: z.number().int().min(1),
|
|
35
|
+
})
|
|
36
|
+
.superRefine((target, context) => {
|
|
37
|
+
if (target.driver === "mariadb" && !target.databaseName)
|
|
38
|
+
context.addIssue({
|
|
39
|
+
code: "custom",
|
|
40
|
+
path: ["databaseName"],
|
|
41
|
+
message: "Database name is required.",
|
|
42
|
+
});
|
|
43
|
+
if (target.driver === "sqlite" && !target.sqlitePath)
|
|
44
|
+
context.addIssue({
|
|
45
|
+
code: "custom",
|
|
46
|
+
path: ["sqlitePath"],
|
|
47
|
+
message: "Database path is required.",
|
|
48
|
+
});
|
|
49
|
+
});
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { Generated, Kysely } from "kysely";
|
|
2
|
+
import type { Migration } from "kysely/migration";
|
|
3
|
+
export interface DatabaseInfrastructureSchema {
|
|
4
|
+
transfer_checkpoints: {
|
|
5
|
+
job_id: string;
|
|
6
|
+
source_sha256: string;
|
|
7
|
+
cursor: number;
|
|
8
|
+
version: Generated<number>;
|
|
9
|
+
};
|
|
10
|
+
application_metadata: {
|
|
11
|
+
key: string;
|
|
12
|
+
value: string;
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
export type DatabaseSchema = DatabaseInfrastructureSchema;
|
|
16
|
+
export interface ConnectionTarget {
|
|
17
|
+
key: string;
|
|
18
|
+
driver: "sqlite" | "mariadb";
|
|
19
|
+
databaseName: string | null;
|
|
20
|
+
sqlitePath: string | null;
|
|
21
|
+
version: number;
|
|
22
|
+
}
|
|
23
|
+
export interface DatabaseProviderOptions<Schema> {
|
|
24
|
+
migrations?: Readonly<Record<string, Migration>>;
|
|
25
|
+
seed?: (database: Kysely<Schema>, environment: NodeJS.ProcessEnv) => Promise<void>;
|
|
26
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { readDatabaseConfiguration, databaseCommandSchema, connectionTargetSchema, } from "./database.schema.js";
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { Kysely } from "kysely";
|
|
2
|
+
import { type ConnectionTarget, type DatabaseSchema } from "./common/index.js";
|
|
3
|
+
export declare class DatabaseCapacityError extends Error {
|
|
4
|
+
constructor();
|
|
5
|
+
}
|
|
6
|
+
export declare class DatabaseConnections {
|
|
7
|
+
private readonly environment;
|
|
8
|
+
private readonly configuration;
|
|
9
|
+
private readonly limit;
|
|
10
|
+
private readonly idleMs;
|
|
11
|
+
private master?;
|
|
12
|
+
private readonly pools;
|
|
13
|
+
private verified;
|
|
14
|
+
private closed;
|
|
15
|
+
private closing?;
|
|
16
|
+
private mutation;
|
|
17
|
+
constructor(environment?: NodeJS.ProcessEnv);
|
|
18
|
+
get driver(): "mariadb" | "sqlite";
|
|
19
|
+
getMaster(): Kysely<import("./index.js").DatabaseInfrastructureSchema>;
|
|
20
|
+
withConnection<T>(input: ConnectionTarget, work: (database: Kysely<DatabaseSchema>) => Promise<T>): Promise<T>;
|
|
21
|
+
provision(input: ConnectionTarget): Promise<import("kysely/migration").MigrationResult[]>;
|
|
22
|
+
verify(): Promise<void>;
|
|
23
|
+
list(): ({
|
|
24
|
+
state: string;
|
|
25
|
+
host: string;
|
|
26
|
+
port: number;
|
|
27
|
+
database: string;
|
|
28
|
+
name: string;
|
|
29
|
+
driver: "mariadb" | "sqlite";
|
|
30
|
+
} | {
|
|
31
|
+
state: string;
|
|
32
|
+
path: string;
|
|
33
|
+
name: string;
|
|
34
|
+
driver: "mariadb" | "sqlite";
|
|
35
|
+
})[];
|
|
36
|
+
poolStats(): {
|
|
37
|
+
capacity: number;
|
|
38
|
+
pools: number;
|
|
39
|
+
leased: number;
|
|
40
|
+
};
|
|
41
|
+
close(): Promise<void>;
|
|
42
|
+
private exclusive;
|
|
43
|
+
private acquire;
|
|
44
|
+
private remove;
|
|
45
|
+
}
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
import { ensureMariaDbDatabase } from "./drivers/mariadb/index.js";
|
|
2
|
+
import { createStorageMigrator } from "./database.migration.js";
|
|
3
|
+
import { Kysely, sql } from "kysely";
|
|
4
|
+
import { z } from "zod";
|
|
5
|
+
import { readDatabaseConfiguration, connectionTargetSchema, } from "./common/index.js";
|
|
6
|
+
import { createSqliteDialect } from "./drivers/sqlite/index.js";
|
|
7
|
+
import { createMariaDbDialect, MariaDbCompatibilityPlugin, } from "./drivers/mariadb/index.js";
|
|
8
|
+
export class DatabaseCapacityError extends Error {
|
|
9
|
+
constructor() {
|
|
10
|
+
super("Database connection capacity reached; retry later.");
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
export class DatabaseConnections {
|
|
14
|
+
environment;
|
|
15
|
+
configuration;
|
|
16
|
+
limit;
|
|
17
|
+
idleMs;
|
|
18
|
+
master;
|
|
19
|
+
pools = new Set();
|
|
20
|
+
verified = false;
|
|
21
|
+
closed = false;
|
|
22
|
+
closing;
|
|
23
|
+
// Serialize acquisition/retirement so concurrent requests cannot exceed the pool bound.
|
|
24
|
+
mutation = Promise.resolve();
|
|
25
|
+
constructor(environment = process.env) {
|
|
26
|
+
this.environment = { ...environment };
|
|
27
|
+
this.configuration = readDatabaseConfiguration(this.environment);
|
|
28
|
+
this.limit = z.coerce
|
|
29
|
+
.number()
|
|
30
|
+
.int()
|
|
31
|
+
.min(1)
|
|
32
|
+
.max(100)
|
|
33
|
+
.parse(environment.DB_POOL_CACHE_LIMIT ?? 16);
|
|
34
|
+
this.idleMs = z.coerce
|
|
35
|
+
.number()
|
|
36
|
+
.int()
|
|
37
|
+
.min(0)
|
|
38
|
+
.max(86_400_000)
|
|
39
|
+
.parse(environment.DB_POOL_IDLE_MS ?? 300_000);
|
|
40
|
+
}
|
|
41
|
+
get driver() {
|
|
42
|
+
return this.configuration.driver;
|
|
43
|
+
}
|
|
44
|
+
getMaster() {
|
|
45
|
+
if (this.closed)
|
|
46
|
+
throw new Error("Database connections are closed.");
|
|
47
|
+
if (!this.master) {
|
|
48
|
+
const config = this.configuration;
|
|
49
|
+
this.master = new Kysely({
|
|
50
|
+
dialect: config.driver === "mariadb"
|
|
51
|
+
? createMariaDbDialect(this.environment)
|
|
52
|
+
: createSqliteDialect(config.path),
|
|
53
|
+
plugins: config.driver === "mariadb" ? [new MariaDbCompatibilityPlugin()] : [],
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
return this.master;
|
|
57
|
+
}
|
|
58
|
+
async withConnection(input, work) {
|
|
59
|
+
const target = connectionTargetSchema.parse(input);
|
|
60
|
+
const entry = await this.exclusive(() => this.acquire(target));
|
|
61
|
+
try {
|
|
62
|
+
return await work(entry.database);
|
|
63
|
+
}
|
|
64
|
+
finally {
|
|
65
|
+
await this.exclusive(async () => {
|
|
66
|
+
entry.leases--;
|
|
67
|
+
entry.touched = Date.now();
|
|
68
|
+
if (entry.leases === 0) {
|
|
69
|
+
entry.drained?.();
|
|
70
|
+
if (entry.retired && !this.closed)
|
|
71
|
+
await this.remove(entry);
|
|
72
|
+
}
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
async provision(input) {
|
|
77
|
+
const target = connectionTargetSchema.parse(input);
|
|
78
|
+
if (target.driver === "mariadb")
|
|
79
|
+
await ensureMariaDbDatabase(this.environment, target.databaseName);
|
|
80
|
+
return this.withConnection(target, async (database) => {
|
|
81
|
+
const result = await createStorageMigrator(database).migrateToLatest();
|
|
82
|
+
if (result.error)
|
|
83
|
+
throw result.error;
|
|
84
|
+
return result.results ?? [];
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
async verify() {
|
|
88
|
+
this.verified = false;
|
|
89
|
+
await sql `SELECT 1`.execute(this.getMaster());
|
|
90
|
+
this.verified = true;
|
|
91
|
+
}
|
|
92
|
+
list() {
|
|
93
|
+
const config = this.configuration;
|
|
94
|
+
return [
|
|
95
|
+
{
|
|
96
|
+
name: "master",
|
|
97
|
+
driver: config.driver,
|
|
98
|
+
...(config.driver === "mariadb"
|
|
99
|
+
? {
|
|
100
|
+
host: config.options.host,
|
|
101
|
+
port: config.options.port,
|
|
102
|
+
database: config.options.database,
|
|
103
|
+
}
|
|
104
|
+
: { path: config.path }),
|
|
105
|
+
state: this.closed
|
|
106
|
+
? "closed"
|
|
107
|
+
: this.verified
|
|
108
|
+
? "verified"
|
|
109
|
+
: this.master
|
|
110
|
+
? "initialized"
|
|
111
|
+
: "configured",
|
|
112
|
+
},
|
|
113
|
+
];
|
|
114
|
+
}
|
|
115
|
+
poolStats() {
|
|
116
|
+
return {
|
|
117
|
+
capacity: this.limit,
|
|
118
|
+
pools: this.pools.size,
|
|
119
|
+
leased: [...this.pools].reduce((sum, pool) => sum + pool.leases, 0),
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
async close() {
|
|
123
|
+
if (!this.closing) {
|
|
124
|
+
this.closed = true;
|
|
125
|
+
this.verified = false;
|
|
126
|
+
this.closing = this.exclusive(async () => {
|
|
127
|
+
// Capture entries without waiting inside the mutation lock: releases must still run.
|
|
128
|
+
return [...this.pools];
|
|
129
|
+
}).then(async (entries) => {
|
|
130
|
+
await Promise.all(entries.map(async (entry) => {
|
|
131
|
+
if (entry.leases)
|
|
132
|
+
await new Promise((resolve) => {
|
|
133
|
+
entry.drained = resolve;
|
|
134
|
+
});
|
|
135
|
+
await entry.database.destroy();
|
|
136
|
+
}));
|
|
137
|
+
this.pools.clear();
|
|
138
|
+
await this.master?.destroy();
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
await this.closing;
|
|
142
|
+
}
|
|
143
|
+
exclusive(work) {
|
|
144
|
+
const result = this.mutation.then(work);
|
|
145
|
+
this.mutation = result.then(() => undefined, () => undefined);
|
|
146
|
+
return result;
|
|
147
|
+
}
|
|
148
|
+
async acquire(target) {
|
|
149
|
+
if (this.closed)
|
|
150
|
+
throw new Error("Database connections are closed.");
|
|
151
|
+
const now = Date.now();
|
|
152
|
+
for (const pool of [...this.pools]) {
|
|
153
|
+
if (pool.target.key === target.key &&
|
|
154
|
+
JSON.stringify(pool.target) !== JSON.stringify(target))
|
|
155
|
+
pool.retired = true;
|
|
156
|
+
if (pool.leases === 0 &&
|
|
157
|
+
(pool.retired || now - pool.touched >= this.idleMs))
|
|
158
|
+
await this.remove(pool);
|
|
159
|
+
}
|
|
160
|
+
const existing = [...this.pools].find((pool) => !pool.retired && JSON.stringify(pool.target) === JSON.stringify(target));
|
|
161
|
+
if (existing) {
|
|
162
|
+
existing.leases++;
|
|
163
|
+
return existing;
|
|
164
|
+
}
|
|
165
|
+
if (this.pools.size >= this.limit) {
|
|
166
|
+
const idle = [...this.pools]
|
|
167
|
+
.filter((pool) => pool.leases === 0)
|
|
168
|
+
.sort((a, b) => a.touched - b.touched)[0];
|
|
169
|
+
if (!idle)
|
|
170
|
+
throw new DatabaseCapacityError();
|
|
171
|
+
await this.remove(idle);
|
|
172
|
+
}
|
|
173
|
+
const database = new Kysely({
|
|
174
|
+
dialect: target.driver === "mariadb"
|
|
175
|
+
? createMariaDbDialect({
|
|
176
|
+
...this.environment,
|
|
177
|
+
DB_MASTER_NAME: target.databaseName,
|
|
178
|
+
})
|
|
179
|
+
: createSqliteDialect(target.sqlitePath),
|
|
180
|
+
plugins: target.driver === "mariadb" ? [new MariaDbCompatibilityPlugin()] : [],
|
|
181
|
+
});
|
|
182
|
+
const entry = {
|
|
183
|
+
target,
|
|
184
|
+
database,
|
|
185
|
+
leases: 1,
|
|
186
|
+
touched: now,
|
|
187
|
+
retired: false,
|
|
188
|
+
};
|
|
189
|
+
this.pools.add(entry);
|
|
190
|
+
return entry;
|
|
191
|
+
}
|
|
192
|
+
async remove(pool) {
|
|
193
|
+
await pool.database.destroy();
|
|
194
|
+
this.pools.delete(pool);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { Kysely, Transaction } from "kysely";
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
import type { DatabaseSchema } from "./common/index.js";
|
|
4
|
+
export declare class DatabaseValidationError extends Error {
|
|
5
|
+
readonly fields: Record<string, string[]>;
|
|
6
|
+
constructor(error: z.ZodError);
|
|
7
|
+
}
|
|
8
|
+
/** Transport and domain schemas remain owned by the calling module. */
|
|
9
|
+
export declare class DatabaseExecution<Schema = DatabaseSchema> {
|
|
10
|
+
private readonly database;
|
|
11
|
+
constructor(database: Kysely<Schema>);
|
|
12
|
+
persist<Input, Output>(schema: z.ZodType<Input>, input: unknown, write: (transaction: Transaction<Schema>, value: Input) => Promise<Output>): Promise<Output>;
|
|
13
|
+
fetch<Row>(input: unknown, read: (database: Kysely<Schema>, page: {
|
|
14
|
+
limit: number;
|
|
15
|
+
offset: number;
|
|
16
|
+
}) => Promise<Row[]>, outputSchema: z.ZodType<Row>): Promise<{
|
|
17
|
+
data: Row[];
|
|
18
|
+
meta: {
|
|
19
|
+
page: number;
|
|
20
|
+
pageSize: number;
|
|
21
|
+
};
|
|
22
|
+
}>;
|
|
23
|
+
/** One transaction: a late invalid row or failed write rolls back every batch. */
|
|
24
|
+
transfer<Row>(schema: z.ZodType<Row>, source: AsyncIterable<unknown> | Iterable<unknown>, write: (transaction: Transaction<Schema>, rows: readonly Row[], signal: AbortSignal) => Promise<void>, batchSize?: number, options?: {
|
|
25
|
+
signal?: AbortSignal;
|
|
26
|
+
timeoutMs?: number;
|
|
27
|
+
maxRows?: number;
|
|
28
|
+
}): Promise<number>;
|
|
29
|
+
}
|
|
30
|
+
export declare function iterateTransfer(source: AsyncIterable<unknown> | Iterable<unknown>, signal: AbortSignal): AsyncGenerator<unknown, void, unknown>;
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
const pageSchema = z.strictObject({
|
|
3
|
+
page: z.number().int().min(1).max(1_000_000),
|
|
4
|
+
pageSize: z.number().int().min(1).max(500),
|
|
5
|
+
});
|
|
6
|
+
const batchSchema = z.number().int().min(1).max(500);
|
|
7
|
+
export class DatabaseValidationError extends Error {
|
|
8
|
+
fields;
|
|
9
|
+
constructor(error) {
|
|
10
|
+
super("Database input validation failed.");
|
|
11
|
+
this.name = "DatabaseValidationError";
|
|
12
|
+
this.fields = Object.create(null);
|
|
13
|
+
for (const issue of error.issues) {
|
|
14
|
+
const key = issue.path.join(".") || "input";
|
|
15
|
+
(this.fields[key] ??= []).push("Invalid value.");
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
/** Transport and domain schemas remain owned by the calling module. */
|
|
20
|
+
export class DatabaseExecution {
|
|
21
|
+
database;
|
|
22
|
+
constructor(database) {
|
|
23
|
+
this.database = database;
|
|
24
|
+
}
|
|
25
|
+
async persist(schema, input, write) {
|
|
26
|
+
const value = validate(schema, input);
|
|
27
|
+
return this.database
|
|
28
|
+
.transaction()
|
|
29
|
+
.execute((transaction) => write(transaction, value));
|
|
30
|
+
}
|
|
31
|
+
async fetch(input, read, outputSchema) {
|
|
32
|
+
const { page, pageSize } = validate(pageSchema, input);
|
|
33
|
+
const rows = await read(this.database, {
|
|
34
|
+
limit: pageSize,
|
|
35
|
+
offset: (page - 1) * pageSize,
|
|
36
|
+
});
|
|
37
|
+
if (!Array.isArray(rows) || rows.length > pageSize)
|
|
38
|
+
throw new Error("Repository exceeded the requested page size.");
|
|
39
|
+
return {
|
|
40
|
+
data: rows.map((row) => validate(outputSchema, row)),
|
|
41
|
+
meta: { page, pageSize },
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
/** One transaction: a late invalid row or failed write rolls back every batch. */
|
|
45
|
+
async transfer(schema, source, write, batchSize = 100, options = {}) {
|
|
46
|
+
validate(batchSchema, batchSize);
|
|
47
|
+
const limits = validate(z.strictObject({
|
|
48
|
+
timeoutMs: z.number().int().min(1).max(3_600_000).default(30_000),
|
|
49
|
+
maxRows: z.number().int().min(1).max(100_000_000).default(1_000_000),
|
|
50
|
+
}), { timeoutMs: options.timeoutMs, maxRows: options.maxRows });
|
|
51
|
+
const timeout = AbortSignal.timeout(limits.timeoutMs);
|
|
52
|
+
const signal = options.signal
|
|
53
|
+
? AbortSignal.any([options.signal, timeout])
|
|
54
|
+
: timeout;
|
|
55
|
+
signal.throwIfAborted();
|
|
56
|
+
return this.database.transaction().execute(async (transaction) => {
|
|
57
|
+
let count = 0;
|
|
58
|
+
let batch = [];
|
|
59
|
+
for await (const input of iterateTransfer(source, signal)) {
|
|
60
|
+
if (count + batch.length >= limits.maxRows)
|
|
61
|
+
throw new Error("Transfer row limit exceeded.");
|
|
62
|
+
batch.push(validate(schema, input));
|
|
63
|
+
if (batch.length === batchSize) {
|
|
64
|
+
await write(transaction, batch, signal);
|
|
65
|
+
signal.throwIfAborted();
|
|
66
|
+
count += batch.length;
|
|
67
|
+
batch = [];
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
if (batch.length) {
|
|
71
|
+
await write(transaction, batch, signal);
|
|
72
|
+
signal.throwIfAborted();
|
|
73
|
+
count += batch.length;
|
|
74
|
+
}
|
|
75
|
+
return count;
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
function validate(schema, input) {
|
|
80
|
+
const result = schema.safeParse(input);
|
|
81
|
+
if (!result.success)
|
|
82
|
+
throw new DatabaseValidationError(result.error);
|
|
83
|
+
return result.data;
|
|
84
|
+
}
|
|
85
|
+
export async function* iterateTransfer(source, signal) {
|
|
86
|
+
const iterator = Symbol.asyncIterator in source
|
|
87
|
+
? source[Symbol.asyncIterator]()
|
|
88
|
+
: source[Symbol.iterator]();
|
|
89
|
+
try {
|
|
90
|
+
while (true) {
|
|
91
|
+
signal.throwIfAborted();
|
|
92
|
+
const next = await new Promise((resolve, reject) => {
|
|
93
|
+
const abort = () => reject(signal.reason);
|
|
94
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
95
|
+
Promise.resolve()
|
|
96
|
+
.then(() => iterator.next())
|
|
97
|
+
.then(resolve, reject)
|
|
98
|
+
.finally(() => signal.removeEventListener("abort", abort));
|
|
99
|
+
});
|
|
100
|
+
if (next.done)
|
|
101
|
+
return;
|
|
102
|
+
yield next.value;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
finally {
|
|
106
|
+
// A stalled producer must not hold the transaction while its cleanup waits.
|
|
107
|
+
Promise.resolve(iterator.return?.()).catch(() => undefined);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { type Kysely } from "kysely";
|
|
2
|
+
import { type Migration, Migrator } from "kysely/migration";
|
|
3
|
+
export declare const transferCheckpointsMigration: Migration;
|
|
4
|
+
export declare const databaseMetadataMigration: Migration;
|
|
5
|
+
export declare function createDatabaseMigrator<Schema>(database: Kysely<Schema>, migrations?: Readonly<Record<string, Migration>>): Migrator;
|
|
6
|
+
export declare function prepareMigrationStorage<Schema>(database: Kysely<Schema>, driver: string): Promise<void>;
|
|
7
|
+
export declare const storageMigrations: {
|
|
8
|
+
"001_database_metadata": Migration;
|
|
9
|
+
"002_transfer_checkpoints": Migration;
|
|
10
|
+
};
|
|
11
|
+
export declare function createStorageMigrator<Schema>(database: Kysely<Schema>): Migrator;
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { sql } from "kysely";
|
|
2
|
+
import { Migrator } from "kysely/migration";
|
|
3
|
+
export const transferCheckpointsMigration = {
|
|
4
|
+
async up(database) {
|
|
5
|
+
await database.schema
|
|
6
|
+
.createTable("transfer_checkpoints")
|
|
7
|
+
.addColumn("job_id", "text", (column) => column.primaryKey())
|
|
8
|
+
.addColumn("source_sha256", "text", (column) => column.notNull())
|
|
9
|
+
.addColumn("cursor", "integer", (column) => column.notNull().defaultTo(0))
|
|
10
|
+
.addColumn("version", "integer", (column) => column.notNull().defaultTo(1))
|
|
11
|
+
.execute();
|
|
12
|
+
},
|
|
13
|
+
async down(database) {
|
|
14
|
+
await database.schema.dropTable("transfer_checkpoints").execute();
|
|
15
|
+
},
|
|
16
|
+
};
|
|
17
|
+
export const databaseMetadataMigration = {
|
|
18
|
+
async up(database) {
|
|
19
|
+
await database.schema
|
|
20
|
+
.createTable("application_metadata")
|
|
21
|
+
.addColumn("key", "text", (column) => column.primaryKey())
|
|
22
|
+
.addColumn("value", "text", (column) => column.notNull())
|
|
23
|
+
.execute();
|
|
24
|
+
},
|
|
25
|
+
async down(database) {
|
|
26
|
+
await database.schema.dropTable("application_metadata").execute();
|
|
27
|
+
},
|
|
28
|
+
};
|
|
29
|
+
export function createDatabaseMigrator(database, migrations = storageMigrations) {
|
|
30
|
+
return new Migrator({
|
|
31
|
+
db: database,
|
|
32
|
+
migrationTableName: "migrations",
|
|
33
|
+
migrationLockTableName: "migration_locks",
|
|
34
|
+
provider: {
|
|
35
|
+
async getMigrations() {
|
|
36
|
+
return { ...migrations };
|
|
37
|
+
},
|
|
38
|
+
},
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
export async function prepareMigrationStorage(database, driver) {
|
|
42
|
+
const tables = new Set((await database.introspection.getTables({ withInternalKyselyTables: true })).map((table) => table.name));
|
|
43
|
+
for (const [previous, current] of [
|
|
44
|
+
["kysely_migration", "migrations"],
|
|
45
|
+
["kysely_migration_lock", "migration_locks"],
|
|
46
|
+
]) {
|
|
47
|
+
if (!tables.has(previous))
|
|
48
|
+
continue;
|
|
49
|
+
if (tables.has(current)) {
|
|
50
|
+
const rows = await sql `SELECT * FROM ${sql.table(current)}`.execute(database);
|
|
51
|
+
if (current === "migrations"
|
|
52
|
+
? rows.rows.length > 0
|
|
53
|
+
: rows.rows.some((row) => Number(row.is_locked) !== 0)) {
|
|
54
|
+
throw new Error(`Both ${previous} and ${current} exist; migration history needs reconciliation.`);
|
|
55
|
+
}
|
|
56
|
+
await database.schema.dropTable(current).execute();
|
|
57
|
+
}
|
|
58
|
+
await database.schema.alterTable(previous).renameTo(current).execute();
|
|
59
|
+
tables.add(current);
|
|
60
|
+
}
|
|
61
|
+
if (driver !== "mariadb" || !tables.has("migrations"))
|
|
62
|
+
return;
|
|
63
|
+
const column = await sql `SELECT DATA_TYPE FROM information_schema.COLUMNS
|
|
64
|
+
WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME='migrations' AND COLUMN_NAME='timestamp'`.execute(database);
|
|
65
|
+
if (column.rows[0]?.DATA_TYPE === "datetime")
|
|
66
|
+
return;
|
|
67
|
+
const invalid = await sql `SELECT COUNT(*) total FROM migrations
|
|
68
|
+
WHERE timestamp NOT REGEXP '^[0-9]{4}-[0-9]{2}-[0-9]{2}[T ][0-9]{2}:[0-9]{2}:[0-9]{2}(\\.[0-9]{1,6})?Z?$'`.execute(database);
|
|
69
|
+
if (Number(invalid.rows[0]?.total))
|
|
70
|
+
throw new Error("Migration timestamps contain unsupported values.");
|
|
71
|
+
await sql `UPDATE migrations SET timestamp=REPLACE(REPLACE(timestamp,'T',' '),'Z','')`.execute(database);
|
|
72
|
+
await sql `ALTER TABLE migrations MODIFY COLUMN timestamp DATETIME(3) NOT NULL`.execute(database);
|
|
73
|
+
}
|
|
74
|
+
export const storageMigrations = {
|
|
75
|
+
"001_database_metadata": databaseMetadataMigration,
|
|
76
|
+
"002_transfer_checkpoints": transferCheckpointsMigration,
|
|
77
|
+
};
|
|
78
|
+
export function createStorageMigrator(database) {
|
|
79
|
+
return createDatabaseMigrator(database, storageMigrations);
|
|
80
|
+
}
|