@stonyx/orm 0.3.2-beta.327 → 0.3.2-beta.329
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/config/environment.js +4 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.js +2 -2
- package/dist/main.d.ts +1 -0
- package/dist/manage-record.d.ts +4 -0
- package/dist/manage-record.js +72 -42
- package/dist/mysql/connection.d.ts +2 -0
- package/dist/mysql/mysql-db.d.ts +32 -10
- package/dist/mysql/mysql-db.js +218 -58
- package/dist/mysql/query-builder.d.ts +1 -0
- package/dist/mysql/query-builder.js +17 -0
- package/dist/postgres/postgres-db.d.ts +1 -0
- package/dist/postgres/postgres-db.js +1 -0
- package/package.json +1 -1
- package/src/index.ts +2 -2
- package/src/main.ts +1 -0
- package/src/manage-record.ts +89 -42
- package/src/mysql/connection.ts +2 -0
- package/src/mysql/mysql-db.ts +242 -56
- package/src/mysql/query-builder.ts +21 -0
- package/src/postgres/postgres-db.ts +1 -0
package/README.md
CHANGED
|
@@ -1057,6 +1057,8 @@ test('validation hook rejects negative age', async () => {
|
|
|
1057
1057
|
| `hasMany` | Define a one-to-many relationship. |
|
|
1058
1058
|
| `createRecord` | Instantiate a record with proper serialization and relationships. |
|
|
1059
1059
|
| `updateRecord` | Update an existing record with new data. |
|
|
1060
|
+
| `createRecordAndPersist` | `createRecord`, awaited: resolves once the write has landed, rejects with its error. Not on DynamoDB yet (#299). |
|
|
1061
|
+
| `updateRecordAndPersist` | `updateRecord`, awaited: resolves once the write has landed, rejects with its error. Not on DynamoDB yet (#299). |
|
|
1060
1062
|
| `store` | Singleton store for all model instances. |
|
|
1061
1063
|
| `relationships` | Access all relationships (`hasMany`, `belongsTo`, `global`, `pending`). |
|
|
1062
1064
|
| `beforeHook` | Register a before hook that can halt operations. |
|
package/config/environment.js
CHANGED
|
@@ -19,6 +19,8 @@ const {
|
|
|
19
19
|
MYSQL_DATABASE,
|
|
20
20
|
MYSQL_CONNECTION_LIMIT,
|
|
21
21
|
MYSQL_MIGRATIONS_DIR,
|
|
22
|
+
MYSQL_WRITE_BATCH_SIZE,
|
|
23
|
+
MYSQL_WRITE_QUEUE_LIMIT,
|
|
22
24
|
PG_HOST,
|
|
23
25
|
PG_PORT,
|
|
24
26
|
PG_USER,
|
|
@@ -70,6 +72,8 @@ export default {
|
|
|
70
72
|
connectionLimit: parseInt(MYSQL_CONNECTION_LIMIT ?? '10'),
|
|
71
73
|
migrationsDir: MYSQL_MIGRATIONS_DIR ?? 'migrations',
|
|
72
74
|
migrationsTable: '__migrations',
|
|
75
|
+
writeBatchSize: parseInt(MYSQL_WRITE_BATCH_SIZE ?? '500'),
|
|
76
|
+
writeQueueLimit: parseInt(MYSQL_WRITE_QUEUE_LIMIT ?? '10000'),
|
|
73
77
|
} : undefined,
|
|
74
78
|
postgres: PG_HOST ? {
|
|
75
79
|
host: PG_HOST ?? 'localhost',
|
package/dist/index.d.ts
CHANGED
|
@@ -4,13 +4,13 @@ import Serializer from './serializer.js';
|
|
|
4
4
|
import attr from './attr.js';
|
|
5
5
|
import belongsTo from './belongs-to.js';
|
|
6
6
|
import hasMany from './has-many.js';
|
|
7
|
-
import { createRecord, updateRecord } from './manage-record.js';
|
|
7
|
+
import { createRecord, updateRecord, createRecordAndPersist, updateRecordAndPersist } from './manage-record.js';
|
|
8
8
|
import { count, avg, sum, min, max } from './aggregates.js';
|
|
9
9
|
export { default } from './main.js';
|
|
10
10
|
export { store, relationships } from './main.js';
|
|
11
11
|
export type { PersistErrorDetail } from './main.js';
|
|
12
12
|
export { Model, View, Serializer };
|
|
13
|
-
export { attr, belongsTo, hasMany, createRecord, updateRecord };
|
|
13
|
+
export { attr, belongsTo, hasMany, createRecord, updateRecord, createRecordAndPersist, updateRecordAndPersist };
|
|
14
14
|
export { count, avg, sum, min, max };
|
|
15
15
|
export { default as normalizeRecordId } from './normalize-record-id.js';
|
|
16
16
|
export type { OrmRequest$ as OrmAccessRequest } from './orm-request.js';
|
package/dist/index.js
CHANGED
|
@@ -19,12 +19,12 @@ import Serializer from './serializer.js';
|
|
|
19
19
|
import attr from './attr.js';
|
|
20
20
|
import belongsTo from './belongs-to.js';
|
|
21
21
|
import hasMany from './has-many.js';
|
|
22
|
-
import { createRecord, updateRecord } from './manage-record.js';
|
|
22
|
+
import { createRecord, updateRecord, createRecordAndPersist, updateRecordAndPersist } from './manage-record.js';
|
|
23
23
|
import { count, avg, sum, min, max } from './aggregates.js';
|
|
24
24
|
export { default } from './main.js';
|
|
25
25
|
export { store, relationships } from './main.js';
|
|
26
26
|
export { Model, View, Serializer }; // base classes
|
|
27
|
-
export { attr, belongsTo, hasMany, createRecord, updateRecord };
|
|
27
|
+
export { attr, belongsTo, hasMany, createRecord, updateRecord, createRecordAndPersist, updateRecordAndPersist };
|
|
28
28
|
export { count, avg, sum, min, max }; // aggregate helpers
|
|
29
29
|
export { default as normalizeRecordId } from './normalize-record-id.js'; // canonical URL-id -> record-id resolution (#270)
|
|
30
30
|
export { beforeHook, afterHook, clearHook, clearAllHooks } from './hooks.js'; // middleware hooks
|
package/dist/main.d.ts
CHANGED
|
@@ -7,6 +7,7 @@ export interface SqlDb {
|
|
|
7
7
|
startup(): Promise<void>;
|
|
8
8
|
shutdown(): Promise<void>;
|
|
9
9
|
persist(operation: string, model: string, context: unknown, response: unknown): Promise<void>;
|
|
10
|
+
persistResolvesAfterWrite?: boolean;
|
|
10
11
|
findRecord(modelName: string, id: unknown): Promise<unknown>;
|
|
11
12
|
findAll(modelName: string, conditions?: Record<string, unknown>): Promise<unknown[]>;
|
|
12
13
|
}
|
package/dist/manage-record.d.ts
CHANGED
|
@@ -11,4 +11,8 @@ export declare function createRecord(modelName: string, rawData?: {
|
|
|
11
11
|
[key: string]: unknown;
|
|
12
12
|
}, userOptions?: CreateRecordOptions): OrmRecord;
|
|
13
13
|
export declare function updateRecord(record: OrmRecord, rawData: unknown, userOptions?: CreateRecordOptions): void;
|
|
14
|
+
export declare function createRecordAndPersist(modelName: string, rawData?: {
|
|
15
|
+
[key: string]: unknown;
|
|
16
|
+
}, userOptions?: CreateRecordOptions): Promise<OrmRecord>;
|
|
17
|
+
export declare function updateRecordAndPersist(record: OrmRecord, rawData: unknown, userOptions?: CreateRecordOptions): Promise<void>;
|
|
14
18
|
export {};
|
package/dist/manage-record.js
CHANGED
|
@@ -8,7 +8,57 @@ const defaultOptions = {
|
|
|
8
8
|
transform: true
|
|
9
9
|
};
|
|
10
10
|
let pendingIdCounter = 0;
|
|
11
|
+
const writeNotRequested = Promise.resolve();
|
|
12
|
+
function ignoreReportedFailure() { }
|
|
13
|
+
function persistRequested(options, userOptions) {
|
|
14
|
+
return !options.isDbRecord && !userOptions._relationshipKey && !options._skipAutoPersist;
|
|
15
|
+
}
|
|
16
|
+
function reportPersistFailure(operation, modelName, recordId, failure) {
|
|
17
|
+
const error = failure instanceof Error ? failure : new Error(String(failure));
|
|
18
|
+
Orm.instance.emitPersistError({ operation, modelName, recordId, error });
|
|
19
|
+
return error;
|
|
20
|
+
}
|
|
11
21
|
export function createRecord(modelName, rawData = {}, userOptions = {}) {
|
|
22
|
+
const { result: record, written } = createRecordWrite(modelName, rawData, userOptions);
|
|
23
|
+
written.catch(ignoreReportedFailure);
|
|
24
|
+
return record;
|
|
25
|
+
}
|
|
26
|
+
export function updateRecord(record, rawData, userOptions = {}) {
|
|
27
|
+
updateRecordWrite(record, rawData, userOptions).written.catch(ignoreReportedFailure);
|
|
28
|
+
}
|
|
29
|
+
export async function createRecordAndPersist(modelName, rawData = {}, userOptions = {}) {
|
|
30
|
+
assertWriteCanBeAwaited();
|
|
31
|
+
const { result: record, written } = createRecordWrite(modelName, rawData, userOptions);
|
|
32
|
+
await written;
|
|
33
|
+
await saveJsonDatabase('create', modelName, record.id, { ...defaultOptions, ...userOptions }, userOptions);
|
|
34
|
+
return record;
|
|
35
|
+
}
|
|
36
|
+
export async function updateRecordAndPersist(record, rawData, userOptions = {}) {
|
|
37
|
+
assertWriteCanBeAwaited();
|
|
38
|
+
await updateRecordWrite(record, rawData, userOptions).written;
|
|
39
|
+
await saveJsonDatabase('update', record?.__model?.__name ?? '', record.id, { ...defaultOptions, ...userOptions }, userOptions);
|
|
40
|
+
}
|
|
41
|
+
function assertWriteCanBeAwaited() {
|
|
42
|
+
const sqlDb = Orm.instance?.sqlDb;
|
|
43
|
+
if (sqlDb && sqlDb.persistResolvesAfterWrite !== true) {
|
|
44
|
+
throw new Error('The configured database adapter cannot confirm a write has landed, so createRecordAndPersist/updateRecordAndPersist are unavailable for it (DynamoDB: abofs/stonyx-orm#299)');
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
async function saveJsonDatabase(operation, modelName, recordId, options, userOptions) {
|
|
48
|
+
const orm = Orm.instance;
|
|
49
|
+
if (orm.sqlDb || !persistRequested(options, userOptions))
|
|
50
|
+
return;
|
|
51
|
+
const jsonDatabase = orm.db;
|
|
52
|
+
if (!jsonDatabase?.save)
|
|
53
|
+
return;
|
|
54
|
+
try {
|
|
55
|
+
await jsonDatabase.save();
|
|
56
|
+
}
|
|
57
|
+
catch (failure) {
|
|
58
|
+
throw reportPersistFailure(operation, modelName, recordId, failure);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
function createRecordWrite(modelName, rawData, userOptions) {
|
|
12
62
|
const orm = Orm.instance;
|
|
13
63
|
const { initialized } = Orm;
|
|
14
64
|
const options = { ...defaultOptions, ...userOptions };
|
|
@@ -26,9 +76,7 @@ export function createRecord(modelName, rawData = {}, userOptions = {}) {
|
|
|
26
76
|
assignRecordId(modelName, rawData);
|
|
27
77
|
const existingRecord = modelStore.get(rawData.id);
|
|
28
78
|
if (existingRecord instanceof OrmRecord) {
|
|
29
|
-
|
|
30
|
-
updateRecord(existingRecord, rawData, { ...options, update: true });
|
|
31
|
-
return existingRecord;
|
|
79
|
+
return { result: existingRecord, written: updateRecordWrite(existingRecord, rawData, { ...options, update: true }).written };
|
|
32
80
|
}
|
|
33
81
|
const recordClasses = orm.getRecordClasses(modelName);
|
|
34
82
|
const modelClass = recordClasses.modelClass;
|
|
@@ -101,56 +149,38 @@ export function createRecord(modelName, rawData = {}, userOptions = {}) {
|
|
|
101
149
|
// Clear the pending queue
|
|
102
150
|
pendingBelongsTo.length = 0;
|
|
103
151
|
}
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
});
|
|
119
|
-
})
|
|
120
|
-
.finally(() => {
|
|
121
|
-
// Evict non-memory records after persist to prevent unbounded heap growth (stonyx#81)
|
|
122
|
-
if (store._memoryResolver && !store._memoryResolver(modelName)) {
|
|
123
|
-
store.evictRecord(modelName, record.id, registryId);
|
|
124
|
-
}
|
|
125
|
-
});
|
|
126
|
-
}
|
|
127
|
-
return record;
|
|
152
|
+
if (!orm?.sqlDb || !persistRequested(options, userOptions))
|
|
153
|
+
return { result: record, written: writeNotRequested };
|
|
154
|
+
const registryIdBeforePersist = record.id;
|
|
155
|
+
const response = { data: { id: record.id } };
|
|
156
|
+
const written = orm.sqlDb.persist('create', modelName, { rawData }, response)
|
|
157
|
+
.catch((failure) => {
|
|
158
|
+
throw reportPersistFailure('create', modelName, record.id, failure);
|
|
159
|
+
})
|
|
160
|
+
.finally(() => {
|
|
161
|
+
if (store._memoryResolver && !store._memoryResolver(modelName)) {
|
|
162
|
+
store.evictRecord(modelName, record.id, registryIdBeforePersist);
|
|
163
|
+
}
|
|
164
|
+
});
|
|
165
|
+
return { result: record, written };
|
|
128
166
|
}
|
|
129
|
-
|
|
167
|
+
function updateRecordWrite(record, rawData, userOptions) {
|
|
130
168
|
if (!rawData)
|
|
131
169
|
throw new Error('rawData must be passed in to updateRecord call');
|
|
132
|
-
// Guard: read-only views cannot be updated
|
|
133
170
|
const modelName = record?.__model?.__name;
|
|
134
171
|
if (modelName && Orm.instance?.isView?.(modelName)) {
|
|
135
172
|
throw new Error(`Cannot update records for read-only view '${modelName}'`);
|
|
136
173
|
}
|
|
137
174
|
const options = { ...defaultOptions, ...userOptions, update: true };
|
|
138
|
-
// Capture old state before update for SQL diff
|
|
139
175
|
const oldState = record.__data ? JSON.parse(JSON.stringify(record.__data)) : {};
|
|
140
176
|
record.serialize(rawData, options);
|
|
141
|
-
// Auto-persist to SQL — skip for DB loads (isDbRecord) and relationship resolution (_relationshipKey)
|
|
142
177
|
const orm = Orm.instance;
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
recordId: record.id,
|
|
150
|
-
error: err instanceof Error ? err : new Error(String(err)),
|
|
151
|
-
});
|
|
152
|
-
});
|
|
153
|
-
}
|
|
178
|
+
if (!orm?.sqlDb || !modelName || !persistRequested(options, userOptions))
|
|
179
|
+
return { result: undefined, written: writeNotRequested };
|
|
180
|
+
const written = orm.sqlDb.persist('update', modelName, { record, oldState }, {}).catch((failure) => {
|
|
181
|
+
throw reportPersistFailure('update', modelName, record.id, failure);
|
|
182
|
+
});
|
|
183
|
+
return { result: undefined, written };
|
|
154
184
|
}
|
|
155
185
|
/**
|
|
156
186
|
* gets the next available id based on last record entry.
|
|
@@ -9,6 +9,8 @@ interface MysqlConfig {
|
|
|
9
9
|
migrationsTable?: string;
|
|
10
10
|
migrationsDir?: string;
|
|
11
11
|
autoMigrate?: boolean;
|
|
12
|
+
writeBatchSize?: number;
|
|
13
|
+
writeQueueLimit?: number;
|
|
12
14
|
}
|
|
13
15
|
export declare function getPool(mysqlConfig: MysqlConfig): Promise<Pool>;
|
|
14
16
|
export declare function closePool(): Promise<void>;
|
package/dist/mysql/mysql-db.d.ts
CHANGED
|
@@ -3,7 +3,7 @@ import type { MysqlConfig } from './connection.js';
|
|
|
3
3
|
import { ensureMigrationsTable, getAppliedMigrations, getMigrationFiles, applyMigration, parseMigrationFile } from './migration-runner.js';
|
|
4
4
|
import { introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot } from './schema-introspector.js';
|
|
5
5
|
import { loadLatestSnapshot, detectSchemaDrift } from './migration-generator.js';
|
|
6
|
-
import { buildInsert, buildUpdate, buildDelete, buildSelect } from './query-builder.js';
|
|
6
|
+
import { buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect } from './query-builder.js';
|
|
7
7
|
import { createRecord } from '../manage-record.js';
|
|
8
8
|
import { confirm } from '@stonyx/utils/prompt';
|
|
9
9
|
import { readFile } from '@stonyx/utils/file';
|
|
@@ -23,6 +23,13 @@ interface PersistResponse {
|
|
|
23
23
|
id?: unknown;
|
|
24
24
|
};
|
|
25
25
|
}
|
|
26
|
+
type ExitEmitter = Pick<NodeJS.Process, 'once' | 'removeListener'>;
|
|
27
|
+
export declare class WriteQueueFullError extends Error {
|
|
28
|
+
constructor(limit: number);
|
|
29
|
+
}
|
|
30
|
+
export declare class WriteQueueClosedError extends Error {
|
|
31
|
+
constructor();
|
|
32
|
+
}
|
|
26
33
|
interface OrmStore {
|
|
27
34
|
get(key: string): Map<number | string, unknown> | undefined;
|
|
28
35
|
get(key: string, id: number | string): unknown;
|
|
@@ -44,6 +51,7 @@ interface MysqlDBDeps {
|
|
|
44
51
|
loadLatestSnapshot: typeof loadLatestSnapshot;
|
|
45
52
|
detectSchemaDrift: typeof detectSchemaDrift;
|
|
46
53
|
buildInsert: typeof buildInsert;
|
|
54
|
+
buildInsertMany: typeof buildInsertMany;
|
|
47
55
|
buildUpdate: typeof buildUpdate;
|
|
48
56
|
buildDelete: typeof buildDelete;
|
|
49
57
|
buildSelect: typeof buildSelect;
|
|
@@ -55,25 +63,30 @@ interface MysqlDBDeps {
|
|
|
55
63
|
config: typeof config;
|
|
56
64
|
log: Record<string, ((...args: unknown[]) => void) | undefined>;
|
|
57
65
|
path: typeof path;
|
|
66
|
+
process: ExitEmitter;
|
|
58
67
|
}
|
|
59
68
|
export default class MysqlDB {
|
|
60
69
|
static instance: MysqlDB | undefined;
|
|
61
70
|
deps: MysqlDBDeps;
|
|
62
71
|
pool: Pool | null;
|
|
63
72
|
mysqlConfig: MysqlConfig;
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
*/
|
|
71
|
-
private _writeQueue;
|
|
73
|
+
readonly persistResolvesAfterWrite = true;
|
|
74
|
+
private _queuedWrites;
|
|
75
|
+
private _inFlightWrites;
|
|
76
|
+
private _drainingWrites;
|
|
77
|
+
private _acceptingWrites;
|
|
78
|
+
private _exitReport;
|
|
72
79
|
constructor(deps?: Partial<MysqlDBDeps>);
|
|
73
80
|
private requirePool;
|
|
74
81
|
init(): Promise<void>;
|
|
82
|
+
get writeBatchSize(): number;
|
|
83
|
+
get writeQueueLimit(): number;
|
|
84
|
+
get pendingWriteCount(): number;
|
|
75
85
|
startup(): Promise<void>;
|
|
76
86
|
shutdown(): Promise<void>;
|
|
87
|
+
private _registerExitReport;
|
|
88
|
+
private _unregisterExitReport;
|
|
89
|
+
private _reportUnpersistedWrites;
|
|
77
90
|
save(): Promise<void>;
|
|
78
91
|
/**
|
|
79
92
|
* Loads only models with memory: true into the in-memory store on startup.
|
|
@@ -101,7 +114,16 @@ export default class MysqlDB {
|
|
|
101
114
|
private _evictIfNotMemory;
|
|
102
115
|
private _rowToRawData;
|
|
103
116
|
persist(operation: string, modelName: string, context: PersistContext, response: PersistResponse): Promise<void>;
|
|
104
|
-
private
|
|
117
|
+
private _drainWritesInBackground;
|
|
118
|
+
private _drainQueuedWrites;
|
|
119
|
+
private _takeNextWriteGroup;
|
|
120
|
+
private _executeWriteGroup;
|
|
121
|
+
private _persistCreates;
|
|
122
|
+
private _prepareCreate;
|
|
123
|
+
private _chunkForSingleInsert;
|
|
124
|
+
private _insertCreates;
|
|
125
|
+
private _insertCreatesOneByOne;
|
|
126
|
+
private _adoptGeneratedId;
|
|
105
127
|
private _persistUpdate;
|
|
106
128
|
private _persistDelete;
|
|
107
129
|
private _recordToRow;
|
package/dist/mysql/mysql-db.js
CHANGED
|
@@ -2,7 +2,7 @@ import { getPool, closePool } from './connection.js';
|
|
|
2
2
|
import { ensureMigrationsTable, getAppliedMigrations, getMigrationFiles, applyMigration, parseMigrationFile } from './migration-runner.js';
|
|
3
3
|
import { introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot } from './schema-introspector.js';
|
|
4
4
|
import { loadLatestSnapshot, detectSchemaDrift } from './migration-generator.js';
|
|
5
|
-
import { buildInsert, buildUpdate, buildDelete, buildSelect } from './query-builder.js';
|
|
5
|
+
import { buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect } from './query-builder.js';
|
|
6
6
|
import { store } from '@stonyx/orm';
|
|
7
7
|
import { createRecord } from '../manage-record.js';
|
|
8
8
|
import { confirm } from '@stonyx/utils/prompt';
|
|
@@ -12,28 +12,41 @@ import { isDbError } from '../utils.js';
|
|
|
12
12
|
import config from 'stonyx/config';
|
|
13
13
|
import log from 'stonyx/log';
|
|
14
14
|
import path from 'path';
|
|
15
|
+
const MYSQL_MAX_PLACEHOLDERS = 65535;
|
|
16
|
+
const DEFAULT_WRITE_BATCH_SIZE = 500;
|
|
17
|
+
const DEFAULT_WRITE_QUEUE_LIMIT = 10000;
|
|
18
|
+
export class WriteQueueFullError extends Error {
|
|
19
|
+
constructor(limit) {
|
|
20
|
+
super(`MysqlDB write queue is full (${limit} queued writes, writeQueueLimit); the write was rejected`);
|
|
21
|
+
this.name = 'WriteQueueFullError';
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
export class WriteQueueClosedError extends Error {
|
|
25
|
+
constructor() {
|
|
26
|
+
super('MysqlDB is shutting down; the write was rejected');
|
|
27
|
+
this.name = 'WriteQueueClosedError';
|
|
28
|
+
}
|
|
29
|
+
}
|
|
15
30
|
const defaultDeps = {
|
|
16
31
|
getPool, closePool, ensureMigrationsTable, getAppliedMigrations,
|
|
17
32
|
getMigrationFiles, applyMigration, parseMigrationFile,
|
|
18
33
|
introspectModels, introspectViews, getTopologicalOrder, schemasToSnapshot,
|
|
19
34
|
loadLatestSnapshot, detectSchemaDrift,
|
|
20
|
-
buildInsert, buildUpdate, buildDelete, buildSelect,
|
|
35
|
+
buildInsert, buildInsertMany, buildUpdate, buildDelete, buildSelect,
|
|
21
36
|
createRecord, store: store, confirm, readFile, getPluralName,
|
|
22
|
-
config, log, path
|
|
37
|
+
config, log, path, process
|
|
23
38
|
};
|
|
24
39
|
export default class MysqlDB {
|
|
25
40
|
static instance;
|
|
26
41
|
deps;
|
|
27
42
|
pool;
|
|
28
43
|
mysqlConfig;
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
*/
|
|
36
|
-
_writeQueue = Promise.resolve();
|
|
44
|
+
persistResolvesAfterWrite = true;
|
|
45
|
+
_queuedWrites = [];
|
|
46
|
+
_inFlightWrites = new Set();
|
|
47
|
+
_drainingWrites = null;
|
|
48
|
+
_acceptingWrites = true;
|
|
49
|
+
_exitReport = null;
|
|
37
50
|
constructor(deps = {}) {
|
|
38
51
|
if (MysqlDB.instance)
|
|
39
52
|
return MysqlDB.instance;
|
|
@@ -54,6 +67,17 @@ export default class MysqlDB {
|
|
|
54
67
|
this.pool = await this.deps.getPool(this.mysqlConfig);
|
|
55
68
|
await this.deps.ensureMigrationsTable(this.pool, this.mysqlConfig.migrationsTable);
|
|
56
69
|
await this.loadMemoryRecords();
|
|
70
|
+
this._acceptingWrites = true;
|
|
71
|
+
this._registerExitReport();
|
|
72
|
+
}
|
|
73
|
+
get writeBatchSize() {
|
|
74
|
+
return Number(this.mysqlConfig?.writeBatchSize) || DEFAULT_WRITE_BATCH_SIZE;
|
|
75
|
+
}
|
|
76
|
+
get writeQueueLimit() {
|
|
77
|
+
return Number(this.mysqlConfig?.writeQueueLimit) || DEFAULT_WRITE_QUEUE_LIMIT;
|
|
78
|
+
}
|
|
79
|
+
get pendingWriteCount() {
|
|
80
|
+
return this._inFlightWrites.size + this._queuedWrites.length;
|
|
57
81
|
}
|
|
58
82
|
async startup() {
|
|
59
83
|
if (!this.mysqlConfig.migrationsDir)
|
|
@@ -134,9 +158,35 @@ export default class MysqlDB {
|
|
|
134
158
|
}
|
|
135
159
|
}
|
|
136
160
|
async shutdown() {
|
|
161
|
+
this._acceptingWrites = false;
|
|
162
|
+
while (this._drainingWrites)
|
|
163
|
+
await this._drainingWrites;
|
|
164
|
+
this._unregisterExitReport();
|
|
137
165
|
await this.deps.closePool();
|
|
138
166
|
this.pool = null;
|
|
139
167
|
}
|
|
168
|
+
_registerExitReport() {
|
|
169
|
+
if (this._exitReport)
|
|
170
|
+
return;
|
|
171
|
+
this._exitReport = () => this._reportUnpersistedWrites();
|
|
172
|
+
this.deps.process.once('exit', this._exitReport);
|
|
173
|
+
}
|
|
174
|
+
_unregisterExitReport() {
|
|
175
|
+
if (!this._exitReport)
|
|
176
|
+
return;
|
|
177
|
+
this.deps.process.removeListener('exit', this._exitReport);
|
|
178
|
+
this._exitReport = null;
|
|
179
|
+
}
|
|
180
|
+
_reportUnpersistedWrites() {
|
|
181
|
+
const lostWrites = [...this._inFlightWrites, ...this._queuedWrites];
|
|
182
|
+
if (lostWrites.length === 0)
|
|
183
|
+
return;
|
|
184
|
+
const lostByModel = new Map();
|
|
185
|
+
for (const { modelName } of lostWrites)
|
|
186
|
+
lostByModel.set(modelName, (lostByModel.get(modelName) ?? 0) + 1);
|
|
187
|
+
const breakdown = [...lostByModel].map(([modelName, count]) => `${modelName}: ${count}`).join(', ');
|
|
188
|
+
this.deps.log.error?.(`[MysqlDB] Process exiting with ${lostWrites.length} queued write(s) not persisted; they are lost (${breakdown})`);
|
|
189
|
+
}
|
|
140
190
|
async save() {
|
|
141
191
|
// No-op: MySQL persists data immediately via persist()
|
|
142
192
|
}
|
|
@@ -326,64 +376,174 @@ export default class MysqlDB {
|
|
|
326
376
|
return rawData;
|
|
327
377
|
}
|
|
328
378
|
async persist(operation, modelName, context, response) {
|
|
329
|
-
// Views are read-only — no-op for all write operations
|
|
330
379
|
const Orm = (await import('@stonyx/orm')).default;
|
|
331
380
|
if (Orm.instance?.isView?.(modelName))
|
|
332
381
|
return;
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
382
|
+
if (!this._acceptingWrites)
|
|
383
|
+
throw new WriteQueueClosedError();
|
|
384
|
+
if (this._queuedWrites.length >= this.writeQueueLimit)
|
|
385
|
+
throw new WriteQueueFullError(this.writeQueueLimit);
|
|
386
|
+
return new Promise((resolve, reject) => {
|
|
387
|
+
const write = {
|
|
388
|
+
operation, modelName, context, response,
|
|
389
|
+
resolve: () => {
|
|
390
|
+
this._inFlightWrites.delete(write);
|
|
391
|
+
resolve();
|
|
392
|
+
},
|
|
393
|
+
reject: (error) => {
|
|
394
|
+
this._inFlightWrites.delete(write);
|
|
395
|
+
reject(error);
|
|
396
|
+
},
|
|
397
|
+
};
|
|
398
|
+
this._queuedWrites.push(write);
|
|
399
|
+
this._drainWritesInBackground();
|
|
400
|
+
});
|
|
401
|
+
}
|
|
402
|
+
_drainWritesInBackground() {
|
|
403
|
+
if (this._drainingWrites)
|
|
352
404
|
return;
|
|
405
|
+
this._drainingWrites = this._drainQueuedWrites().finally(() => {
|
|
406
|
+
this._drainingWrites = null;
|
|
407
|
+
});
|
|
408
|
+
}
|
|
409
|
+
async _drainQueuedWrites() {
|
|
410
|
+
while (this._queuedWrites.length > 0) {
|
|
411
|
+
const writes = this._takeNextWriteGroup();
|
|
412
|
+
writes.forEach(write => this._inFlightWrites.add(write));
|
|
413
|
+
try {
|
|
414
|
+
await this._executeWriteGroup(writes);
|
|
415
|
+
}
|
|
416
|
+
catch (error) {
|
|
417
|
+
writes.filter(write => this._inFlightWrites.has(write)).forEach(write => write.reject(error));
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
_takeNextWriteGroup() {
|
|
422
|
+
const [first] = this._queuedWrites;
|
|
423
|
+
let groupSize = 1;
|
|
424
|
+
if (first.operation === 'create') {
|
|
425
|
+
while (groupSize < this.writeBatchSize
|
|
426
|
+
&& groupSize < this._queuedWrites.length
|
|
427
|
+
&& this._queuedWrites[groupSize].operation === 'create'
|
|
428
|
+
&& this._queuedWrites[groupSize].modelName === first.modelName)
|
|
429
|
+
groupSize++;
|
|
430
|
+
}
|
|
431
|
+
return this._queuedWrites.splice(0, groupSize);
|
|
432
|
+
}
|
|
433
|
+
async _executeWriteGroup(writes) {
|
|
434
|
+
const [first] = writes;
|
|
435
|
+
if (first.operation === 'create')
|
|
436
|
+
return this._persistCreates(first.modelName, writes);
|
|
437
|
+
try {
|
|
438
|
+
if (first.operation === 'update')
|
|
439
|
+
await this._persistUpdate(first.modelName, first.context, first.response);
|
|
440
|
+
if (first.operation === 'delete')
|
|
441
|
+
await this._persistDelete(first.modelName, first.context);
|
|
442
|
+
first.resolve();
|
|
443
|
+
}
|
|
444
|
+
catch (error) {
|
|
445
|
+
first.reject(error);
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
async _persistCreates(modelName, writes) {
|
|
449
|
+
const schema = this.deps.introspectModels()[modelName];
|
|
450
|
+
const prepared = [];
|
|
451
|
+
for (const write of writes) {
|
|
452
|
+
try {
|
|
453
|
+
const create = schema ? this._prepareCreate(modelName, schema, write) : null;
|
|
454
|
+
if (create)
|
|
455
|
+
prepared.push(create);
|
|
456
|
+
else
|
|
457
|
+
write.resolve();
|
|
458
|
+
}
|
|
459
|
+
catch (error) {
|
|
460
|
+
write.reject(error);
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
for (const chunk of this._chunkForSingleInsert(prepared)) {
|
|
464
|
+
await this._insertCreates(schema, chunk);
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
_prepareCreate(modelName, schema, write) {
|
|
468
|
+
const { context, response } = write;
|
|
353
469
|
const recordId = response?.data?.id;
|
|
354
470
|
const record = recordId != null ? this.deps.store.get(modelName, (isNaN(recordId) ? recordId : parseInt(recordId))) : null;
|
|
355
471
|
if (!record)
|
|
356
|
-
return;
|
|
357
|
-
const
|
|
358
|
-
// For auto-increment models, remove the pending ID.
|
|
359
|
-
// Check context.rawData (not record.__data) because __pendingSqlId is not a model
|
|
360
|
-
// attribute and gets lost during serialization.
|
|
472
|
+
return null;
|
|
473
|
+
const row = this._recordToRow(record, schema);
|
|
361
474
|
const isPendingId = context.rawData?.__pendingSqlId === true;
|
|
362
|
-
if (isPendingId)
|
|
363
|
-
delete
|
|
475
|
+
if (isPendingId)
|
|
476
|
+
delete row.id;
|
|
477
|
+
return { write, modelName, record, row, isPendingId };
|
|
478
|
+
}
|
|
479
|
+
_chunkForSingleInsert(prepared) {
|
|
480
|
+
const chunks = [];
|
|
481
|
+
let chunk = [];
|
|
482
|
+
let chunkColumns = new Set();
|
|
483
|
+
for (const create of prepared) {
|
|
484
|
+
const columnsWithCreate = new Set([...chunkColumns, ...Object.keys(create.row)]);
|
|
485
|
+
const mixesIdSources = chunk.length > 0 && chunk[0].isPendingId !== create.isPendingId;
|
|
486
|
+
const exceedsPlaceholders = (chunk.length + 1) * columnsWithCreate.size > MYSQL_MAX_PLACEHOLDERS;
|
|
487
|
+
if (chunk.length > 0 && (mixesIdSources || exceedsPlaceholders)) {
|
|
488
|
+
chunks.push(chunk);
|
|
489
|
+
chunk = [];
|
|
490
|
+
chunkColumns = new Set(Object.keys(create.row));
|
|
491
|
+
}
|
|
492
|
+
else {
|
|
493
|
+
chunkColumns = columnsWithCreate;
|
|
494
|
+
}
|
|
495
|
+
chunk.push(create);
|
|
364
496
|
}
|
|
365
|
-
|
|
366
|
-
|
|
497
|
+
if (chunk.length > 0)
|
|
498
|
+
chunks.push(chunk);
|
|
499
|
+
return chunks;
|
|
500
|
+
}
|
|
501
|
+
async _insertCreates(schema, creates) {
|
|
502
|
+
if (creates.length === 1)
|
|
503
|
+
return this._insertCreatesOneByOne(schema, creates);
|
|
504
|
+
let firstInsertId;
|
|
505
|
+
try {
|
|
506
|
+
const { sql, values } = this.deps.buildInsertMany(schema.table, creates.map(({ row }) => row));
|
|
507
|
+
const [result] = await this.requirePool().execute(sql, values);
|
|
508
|
+
firstInsertId = result.insertId;
|
|
367
509
|
}
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
// Re-key the record in the store if MySQL generated the ID
|
|
371
|
-
if (isPendingId && result.insertId) {
|
|
372
|
-
const pendingId = record.id;
|
|
373
|
-
const realId = result.insertId;
|
|
374
|
-
const modelStore = this.deps.store.get(modelName);
|
|
375
|
-
if (!modelStore)
|
|
376
|
-
throw new Error(`Model "${modelName}" not found in store during ID re-key`);
|
|
377
|
-
modelStore.delete(pendingId);
|
|
378
|
-
record.__data.id = realId;
|
|
379
|
-
record.id = realId;
|
|
380
|
-
modelStore.set(realId, record);
|
|
381
|
-
// Update the response data with the real ID
|
|
382
|
-
if (response?.data) {
|
|
383
|
-
response.data.id = realId;
|
|
384
|
-
}
|
|
385
|
-
delete record.__data.__pendingSqlId;
|
|
510
|
+
catch {
|
|
511
|
+
return this._insertCreatesOneByOne(schema, creates);
|
|
386
512
|
}
|
|
513
|
+
creates.forEach((create, index) => {
|
|
514
|
+
if (firstInsertId)
|
|
515
|
+
this._adoptGeneratedId(create, firstInsertId + index);
|
|
516
|
+
create.write.resolve();
|
|
517
|
+
});
|
|
518
|
+
}
|
|
519
|
+
async _insertCreatesOneByOne(schema, creates) {
|
|
520
|
+
for (const create of creates) {
|
|
521
|
+
try {
|
|
522
|
+
const { sql, values } = this.deps.buildInsert(schema.table, create.row);
|
|
523
|
+
const [result] = await this.requirePool().execute(sql, values);
|
|
524
|
+
if (result.insertId)
|
|
525
|
+
this._adoptGeneratedId(create, result.insertId);
|
|
526
|
+
create.write.resolve();
|
|
527
|
+
}
|
|
528
|
+
catch (error) {
|
|
529
|
+
create.write.reject(error);
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
_adoptGeneratedId({ modelName, record, isPendingId, write }, realId) {
|
|
534
|
+
if (!isPendingId)
|
|
535
|
+
return;
|
|
536
|
+
const pendingId = record.id;
|
|
537
|
+
const modelStore = this.deps.store.get(modelName);
|
|
538
|
+
if (!modelStore)
|
|
539
|
+
throw new Error(`Model "${modelName}" not found in store during ID re-key`);
|
|
540
|
+
modelStore.delete(pendingId);
|
|
541
|
+
record.__data.id = realId;
|
|
542
|
+
record.id = realId;
|
|
543
|
+
modelStore.set(realId, record);
|
|
544
|
+
if (write.response?.data)
|
|
545
|
+
write.response.data.id = realId;
|
|
546
|
+
delete record.__data.__pendingSqlId;
|
|
387
547
|
}
|
|
388
548
|
async _persistUpdate(modelName, context, response) {
|
|
389
549
|
const schemas = this.deps.introspectModels();
|