metal-orm 1.1.9 → 1.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +769 -764
- package/dist/index.cjs +2255 -284
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +559 -39
- package/dist/index.d.ts +559 -39
- package/dist/index.js +2227 -284
- package/dist/index.js.map +1 -1
- package/package.json +17 -12
- package/scripts/generate-entities/render.mjs +21 -12
- package/scripts/generate-entities/schema.mjs +87 -73
- package/scripts/generate-entities/tree-detection.mjs +67 -61
- package/src/bulk/bulk-context.ts +83 -0
- package/src/bulk/bulk-delete-executor.ts +87 -0
- package/src/bulk/bulk-executor.base.ts +73 -0
- package/src/bulk/bulk-insert-executor.ts +74 -0
- package/src/bulk/bulk-types.ts +70 -0
- package/src/bulk/bulk-update-executor.ts +192 -0
- package/src/bulk/bulk-upsert-executor.ts +93 -0
- package/src/bulk/bulk-utils.ts +91 -0
- package/src/bulk/index.ts +18 -0
- package/src/codegen/typescript.ts +30 -21
- package/src/core/ast/expression-builders.ts +107 -10
- package/src/core/ast/expression-nodes.ts +52 -22
- package/src/core/ast/expression-visitor.ts +23 -13
- package/src/core/ddl/introspect/mysql.ts +113 -36
- package/src/core/dialect/abstract.ts +30 -17
- package/src/core/dialect/mysql/index.ts +20 -5
- package/src/core/execution/db-executor.ts +96 -64
- package/src/core/execution/executors/better-sqlite3-executor.ts +94 -0
- package/src/core/execution/executors/mssql-executor.ts +66 -34
- package/src/core/execution/executors/mysql-executor.ts +98 -66
- package/src/core/execution/executors/postgres-executor.ts +33 -11
- package/src/core/execution/executors/sqlite-executor.ts +86 -30
- package/src/decorators/bootstrap.ts +482 -398
- package/src/decorators/column-decorator.ts +87 -96
- package/src/decorators/decorator-metadata.ts +100 -24
- package/src/decorators/entity.ts +27 -24
- package/src/decorators/relations.ts +231 -149
- package/src/decorators/transformers/transformer-decorators.ts +26 -29
- package/src/decorators/validators/country-validators-decorators.ts +9 -15
- package/src/dto/apply-filter.ts +568 -551
- package/src/index.ts +16 -9
- package/src/orm/entity-hydration.ts +116 -72
- package/src/orm/entity-metadata.ts +347 -301
- package/src/orm/entity-relations.ts +264 -207
- package/src/orm/entity.ts +199 -199
- package/src/orm/execute.ts +13 -13
- package/src/orm/lazy-batch/morph-many.ts +70 -0
- package/src/orm/lazy-batch/morph-one.ts +69 -0
- package/src/orm/lazy-batch/morph-to.ts +59 -0
- package/src/orm/lazy-batch.ts +4 -1
- package/src/orm/orm-session.ts +170 -104
- package/src/orm/pooled-executor-factory.ts +99 -58
- package/src/orm/query-logger.ts +49 -40
- package/src/orm/relation-change-processor.ts +198 -96
- package/src/orm/relations/belongs-to.ts +143 -143
- package/src/orm/relations/has-many.ts +204 -204
- package/src/orm/relations/has-one.ts +174 -174
- package/src/orm/relations/many-to-many.ts +288 -288
- package/src/orm/relations/morph-many.ts +156 -0
- package/src/orm/relations/morph-one.ts +151 -0
- package/src/orm/relations/morph-to.ts +162 -0
- package/src/orm/save-graph.ts +116 -1
- package/src/query-builder/expression-table-mapper.ts +5 -0
- package/src/query-builder/hydration-manager.ts +345 -345
- package/src/query-builder/hydration-planner.ts +178 -148
- package/src/query-builder/relation-conditions.ts +171 -151
- package/src/query-builder/relation-cte-builder.ts +5 -1
- package/src/query-builder/relation-filter-utils.ts +9 -6
- package/src/query-builder/relation-include-strategies.ts +44 -2
- package/src/query-builder/relation-join-strategies.ts +8 -1
- package/src/query-builder/relation-service.ts +250 -241
- package/src/query-builder/select/select-operations.ts +110 -105
- package/src/query-builder/update-include.ts +4 -0
- package/src/schema/relation.ts +296 -188
- package/src/schema/types.ts +138 -123
- package/src/tree/tree-decorator.ts +127 -137
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import type { TableDef } from '../schema/table.js';
|
|
2
|
+
import type { OrmSession } from '../orm/orm-session.js';
|
|
3
|
+
import type { BulkResult, BulkBaseOptions, ChunkOutcome } from './bulk-types.js';
|
|
4
|
+
import { createBulkExecutionContext, type BulkExecutionContext } from './bulk-context.js';
|
|
5
|
+
import {
|
|
6
|
+
splitIntoChunks,
|
|
7
|
+
runWithConcurrency,
|
|
8
|
+
runChunk,
|
|
9
|
+
maybeTransaction,
|
|
10
|
+
aggregateOutcomes,
|
|
11
|
+
aggregateOutcomesWithTimings,
|
|
12
|
+
} from './bulk-utils.js';
|
|
13
|
+
|
|
14
|
+
const DEFAULT_CHUNK_SIZE = 500;
|
|
15
|
+
|
|
16
|
+
export interface BulkExecutorOptions extends BulkBaseOptions {
|
|
17
|
+
chunkSize?: number;
|
|
18
|
+
concurrency?: 'sequential' | number;
|
|
19
|
+
transactional?: boolean;
|
|
20
|
+
timing?: boolean;
|
|
21
|
+
onChunkComplete?: (info: { chunkIndex: number; totalChunks: number; rowsInChunk: number; elapsedMs: number }) => void | Promise<void>;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export abstract class BulkBaseExecutor<TOptions extends BulkExecutorOptions> {
|
|
25
|
+
protected readonly session: OrmSession;
|
|
26
|
+
protected readonly table: TableDef;
|
|
27
|
+
protected readonly ctx: BulkExecutionContext;
|
|
28
|
+
protected readonly options: TOptions;
|
|
29
|
+
protected readonly chunks: unknown[][];
|
|
30
|
+
protected readonly totalChunks: number;
|
|
31
|
+
|
|
32
|
+
constructor(session: OrmSession, table: TableDef, rows: unknown[], options: TOptions = {} as TOptions) {
|
|
33
|
+
this.session = session;
|
|
34
|
+
this.table = table;
|
|
35
|
+
this.ctx = createBulkExecutionContext(session);
|
|
36
|
+
this.options = {
|
|
37
|
+
chunkSize: DEFAULT_CHUNK_SIZE,
|
|
38
|
+
concurrency: 'sequential',
|
|
39
|
+
transactional: true,
|
|
40
|
+
timing: false,
|
|
41
|
+
...options,
|
|
42
|
+
} as TOptions;
|
|
43
|
+
this.chunks = splitIntoChunks(rows, this.options.chunkSize!);
|
|
44
|
+
this.totalChunks = this.chunks.length;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
protected abstract executeChunk(chunk: unknown[], chunkIndex: number): Promise<ChunkOutcome>;
|
|
48
|
+
|
|
49
|
+
async execute(): Promise<BulkResult> {
|
|
50
|
+
const buildTask = (chunk: unknown[], chunkIndex: number) => async (): Promise<ChunkOutcome> => {
|
|
51
|
+
return runChunk(
|
|
52
|
+
() => this.executeChunk(chunk, chunkIndex),
|
|
53
|
+
chunkIndex,
|
|
54
|
+
this.totalChunks,
|
|
55
|
+
chunk.length,
|
|
56
|
+
this.options.timing!,
|
|
57
|
+
this.options.onChunkComplete
|
|
58
|
+
);
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
const tasks = this.chunks.map((chunk, i) => buildTask(chunk, i));
|
|
62
|
+
|
|
63
|
+
const outcomes = await maybeTransaction(
|
|
64
|
+
this.session,
|
|
65
|
+
this.options.transactional!,
|
|
66
|
+
() => runWithConcurrency(tasks, this.options.concurrency!)
|
|
67
|
+
);
|
|
68
|
+
|
|
69
|
+
return this.options.timing
|
|
70
|
+
? aggregateOutcomesWithTimings(outcomes)
|
|
71
|
+
: aggregateOutcomes(outcomes);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { InsertQueryBuilder, ConflictBuilder } from '../query-builder/insert.js';
|
|
2
|
+
import type { TableDef } from '../schema/table.js';
|
|
3
|
+
import type { OrmSession } from '../orm/orm-session.js';
|
|
4
|
+
import type { BulkInsertOptions, ChunkOutcome, InsertRow } from './bulk-types.js';
|
|
5
|
+
import { BulkBaseExecutor, type BulkExecutorOptions } from './bulk-executor.base.js';
|
|
6
|
+
import { resolveReturningColumns, flattenQueryResults, executeCompiled } from './bulk-context.js';
|
|
7
|
+
import type { ValueOperandInput } from '../core/ast/expression.js';
|
|
8
|
+
|
|
9
|
+
interface InsertExecutorOptions extends BulkExecutorOptions {
|
|
10
|
+
returning?: boolean | import('../schema/column-types.js').ColumnDef[];
|
|
11
|
+
onConflict?: import('../core/ast/query.js').UpsertClause;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export class BulkInsertExecutor extends BulkBaseExecutor<InsertExecutorOptions> {
|
|
15
|
+
constructor(
|
|
16
|
+
session: OrmSession,
|
|
17
|
+
table: TableDef,
|
|
18
|
+
rows: InsertRow[],
|
|
19
|
+
options: BulkInsertOptions = {}
|
|
20
|
+
) {
|
|
21
|
+
super(session, table, rows, options);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
protected async executeChunk(chunk: InsertRow[], _chunkIndex: number): Promise<ChunkOutcome> {
|
|
25
|
+
const returningColumns = resolveReturningColumns(this.ctx, this.table, this.options.returning);
|
|
26
|
+
|
|
27
|
+
let builder: InsertQueryBuilder<unknown> | ConflictBuilder<unknown> = new InsertQueryBuilder(this.table).values(chunk);
|
|
28
|
+
|
|
29
|
+
if (this.options.onConflict) {
|
|
30
|
+
const conflictColumns = this.options.onConflict.target?.columns ?? [];
|
|
31
|
+
const conflictBuilder = (builder as InsertQueryBuilder<unknown>).onConflict(conflictColumns);
|
|
32
|
+
|
|
33
|
+
if (this.options.onConflict.action.type === 'DoNothing') {
|
|
34
|
+
builder = conflictBuilder.doNothing();
|
|
35
|
+
} else if (this.options.onConflict.action.type === 'DoUpdate' && this.options.onConflict.action.set) {
|
|
36
|
+
const setMap: Record<string, ValueOperandInput> = {};
|
|
37
|
+
for (const assignment of this.options.onConflict.action.set) {
|
|
38
|
+
const colName = typeof assignment.column === 'object' ? assignment.column.name : assignment.column;
|
|
39
|
+
setMap[colName] = assignment.value;
|
|
40
|
+
}
|
|
41
|
+
builder = conflictBuilder.doUpdate(setMap);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const finalBuilder = builder as InsertQueryBuilder<unknown>;
|
|
46
|
+
|
|
47
|
+
if (returningColumns?.length) {
|
|
48
|
+
finalBuilder.returning(...returningColumns);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const compiled = finalBuilder.compile(this.ctx.dialect);
|
|
52
|
+
const resultSets = await executeCompiled(this.ctx, compiled);
|
|
53
|
+
|
|
54
|
+
return {
|
|
55
|
+
processedRows: chunk.length,
|
|
56
|
+
returning: returningColumns ? flattenQueryResults(resultSets) : [],
|
|
57
|
+
elapsedMs: 0,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export async function bulkInsert<TTable extends TableDef>(
|
|
63
|
+
session: OrmSession,
|
|
64
|
+
table: TTable,
|
|
65
|
+
rows: InsertRow[],
|
|
66
|
+
options: BulkInsertOptions = {}
|
|
67
|
+
): Promise<import('./bulk-types.js').BulkResult> {
|
|
68
|
+
if (!rows.length) {
|
|
69
|
+
return { processedRows: 0, chunksExecuted: 0, returning: [] };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const executor = new BulkInsertExecutor(session, table, rows, options);
|
|
73
|
+
return executor.execute();
|
|
74
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import type { TableDef } from '../schema/table.js';
|
|
2
|
+
import type { OrmSession } from '../orm/orm-session.js';
|
|
3
|
+
import type { ExpressionNode } from '../core/ast/expression-nodes.js';
|
|
4
|
+
import type { ColumnDef } from '../schema/column-types.js';
|
|
5
|
+
import type { UpsertClause } from '../core/ast/query.js';
|
|
6
|
+
import type { ValueOperandInput } from '../core/ast/expression.js';
|
|
7
|
+
|
|
8
|
+
export type { TableDef, OrmSession };
|
|
9
|
+
|
|
10
|
+
export type BulkConcurrency = 'sequential' | number;
|
|
11
|
+
|
|
12
|
+
export interface BulkResult {
|
|
13
|
+
processedRows: number;
|
|
14
|
+
chunksExecuted: number;
|
|
15
|
+
returning: Record<string, unknown>[];
|
|
16
|
+
chunkTimings?: number[];
|
|
17
|
+
metadata?: BulkResultMetadata;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export interface BulkResultMetadata {
|
|
21
|
+
strategy: 'individual' | 'batch' | 'whereIn';
|
|
22
|
+
dialect: string;
|
|
23
|
+
hasReturningSupport: boolean;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export interface BulkBaseOptions {
|
|
27
|
+
chunkSize?: number;
|
|
28
|
+
concurrency?: BulkConcurrency;
|
|
29
|
+
transactional?: boolean;
|
|
30
|
+
timing?: boolean;
|
|
31
|
+
onChunkComplete?: (info: ChunkCompleteInfo) => void | Promise<void>;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export interface ChunkCompleteInfo {
|
|
35
|
+
chunkIndex: number;
|
|
36
|
+
totalChunks: number;
|
|
37
|
+
rowsInChunk: number;
|
|
38
|
+
elapsedMs: number;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export type InsertRow = Record<string, ValueOperandInput>;
|
|
42
|
+
|
|
43
|
+
export interface BulkInsertOptions extends BulkBaseOptions {
|
|
44
|
+
returning?: boolean | ColumnDef[];
|
|
45
|
+
onConflict?: UpsertClause;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export interface BulkUpsertOptions extends BulkInsertOptions {
|
|
49
|
+
conflictColumns?: string[];
|
|
50
|
+
updateColumns?: string[];
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export type UpdateRow = Record<string, ValueOperandInput>;
|
|
54
|
+
|
|
55
|
+
export interface BulkUpdateOptions extends BulkBaseOptions {
|
|
56
|
+
by?: string | string[];
|
|
57
|
+
where?: ExpressionNode;
|
|
58
|
+
returning?: boolean | ColumnDef[];
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export interface BulkDeleteOptions extends BulkBaseOptions {
|
|
62
|
+
by?: string;
|
|
63
|
+
where?: ExpressionNode;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export interface ChunkOutcome {
|
|
67
|
+
processedRows: number;
|
|
68
|
+
returning: Record<string, unknown>[];
|
|
69
|
+
elapsedMs: number;
|
|
70
|
+
}
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
import { UpdateQueryBuilder } from '../query-builder/update.js';
|
|
2
|
+
import { findPrimaryKey } from '../query-builder/hydration-planner.js';
|
|
3
|
+
import { eq, and, inList } from '../core/ast/expression-builders.js';
|
|
4
|
+
import type { TableDef } from '../schema/table.js';
|
|
5
|
+
import type { OrmSession } from '../orm/orm-session.js';
|
|
6
|
+
import type { BulkUpdateOptions, ChunkOutcome, UpdateRow } from './bulk-types.js';
|
|
7
|
+
import type { ValueOperandInput } from '../core/ast/expression.js';
|
|
8
|
+
import type { ExpressionNode } from '../core/ast/expression-nodes.js';
|
|
9
|
+
import { BulkBaseExecutor, type BulkExecutorOptions } from './bulk-executor.base.js';
|
|
10
|
+
import { resolveReturningColumns, flattenQueryResults, executeCompiled, createBulkExecutionContext } from './bulk-context.js';
|
|
11
|
+
import { splitIntoChunks, runWithConcurrency, runChunk, maybeTransaction, aggregateOutcomes, aggregateOutcomesWithTimings } from './bulk-utils.js';
|
|
12
|
+
|
|
13
|
+
interface UpdateExecutorOptions extends BulkExecutorOptions {
|
|
14
|
+
by?: string | string[];
|
|
15
|
+
where?: ExpressionNode;
|
|
16
|
+
returning?: boolean | import('../schema/column-types.js').ColumnDef[];
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function resolveByColumns(table: TableDef, by: string | string[] | undefined): string[] {
|
|
20
|
+
if (!by) return [findPrimaryKey(table)];
|
|
21
|
+
return Array.isArray(by) ? by : [by];
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export class BulkUpdateExecutor extends BulkBaseExecutor<UpdateExecutorOptions> {
|
|
25
|
+
private readonly byColumns: string[];
|
|
26
|
+
|
|
27
|
+
constructor(
|
|
28
|
+
session: OrmSession,
|
|
29
|
+
table: TableDef,
|
|
30
|
+
rows: UpdateRow[],
|
|
31
|
+
options: BulkUpdateOptions = {}
|
|
32
|
+
) {
|
|
33
|
+
super(session, table, rows, options);
|
|
34
|
+
this.byColumns = resolveByColumns(table, options.by);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
protected async executeChunk(chunk: UpdateRow[], _chunkIndex: number): Promise<ChunkOutcome> {
|
|
38
|
+
const allReturning: Record<string, unknown>[] = [];
|
|
39
|
+
const returningColumns = resolveReturningColumns(this.ctx, this.table, this.options.returning);
|
|
40
|
+
const extraWhere = this.options.where;
|
|
41
|
+
|
|
42
|
+
for (const row of chunk) {
|
|
43
|
+
const predicates = this.byColumns.map(colName => {
|
|
44
|
+
const col = this.table.columns[colName];
|
|
45
|
+
if (!col) {
|
|
46
|
+
throw new Error(
|
|
47
|
+
`bulkUpdate: column "${colName}" not found in table "${this.table.name}"`
|
|
48
|
+
);
|
|
49
|
+
}
|
|
50
|
+
const val = row[colName];
|
|
51
|
+
if (val === undefined) {
|
|
52
|
+
throw new Error(
|
|
53
|
+
`bulkUpdate: row is missing the identity column "${colName}" required by the "by" option`
|
|
54
|
+
);
|
|
55
|
+
}
|
|
56
|
+
return eq(col, val as ValueOperandInput);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
const whereExpr: ExpressionNode =
|
|
60
|
+
predicates.length === 1
|
|
61
|
+
? predicates[0]
|
|
62
|
+
: predicates.reduce((acc, p) => and(acc, p) as ExpressionNode, predicates[0]);
|
|
63
|
+
|
|
64
|
+
const finalWhere = extraWhere ? and(whereExpr, extraWhere) as ExpressionNode : whereExpr;
|
|
65
|
+
|
|
66
|
+
const bySet = new Set(this.byColumns);
|
|
67
|
+
const setPayload: Record<string, unknown> = {};
|
|
68
|
+
for (const [key, val] of Object.entries(row)) {
|
|
69
|
+
if (!bySet.has(key) && key in this.table.columns) {
|
|
70
|
+
setPayload[key] = val;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
if (!Object.keys(setPayload).length) continue;
|
|
75
|
+
|
|
76
|
+
let builder = new UpdateQueryBuilder(this.table).set(setPayload).where(finalWhere);
|
|
77
|
+
|
|
78
|
+
if (returningColumns?.length) {
|
|
79
|
+
builder = builder.returning(...returningColumns);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const compiled = builder.compile(this.ctx.dialect);
|
|
83
|
+
const resultSets = await executeCompiled(this.ctx, compiled);
|
|
84
|
+
|
|
85
|
+
if (returningColumns) {
|
|
86
|
+
allReturning.push(...flattenQueryResults(resultSets));
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
return {
|
|
91
|
+
processedRows: chunk.length,
|
|
92
|
+
returning: allReturning,
|
|
93
|
+
elapsedMs: 0,
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export async function bulkUpdate<TTable extends TableDef>(
|
|
99
|
+
session: OrmSession,
|
|
100
|
+
table: TTable,
|
|
101
|
+
rows: UpdateRow[],
|
|
102
|
+
options: BulkUpdateOptions = {}
|
|
103
|
+
): Promise<import('./bulk-types.js').BulkResult> {
|
|
104
|
+
if (!rows.length) {
|
|
105
|
+
return { processedRows: 0, chunksExecuted: 0, returning: [] };
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
const executor = new BulkUpdateExecutor(session, table, rows, options);
|
|
109
|
+
return executor.execute();
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
const DEFAULT_BULK_UPDATE_WHERE_CHUNK_SIZE = 500;
|
|
113
|
+
|
|
114
|
+
export async function bulkUpdateWhere<TTable extends TableDef>(
|
|
115
|
+
session: OrmSession,
|
|
116
|
+
table: TTable,
|
|
117
|
+
ids: ValueOperandInput[],
|
|
118
|
+
set: Record<string, ValueOperandInput>,
|
|
119
|
+
options: Omit<BulkUpdateOptions, 'by' | 'returning'> & {
|
|
120
|
+
by?: string;
|
|
121
|
+
returning?: boolean | import('../schema/column-types.js').ColumnDef[];
|
|
122
|
+
} = {}
|
|
123
|
+
): Promise<import('./bulk-types.js').BulkResult> {
|
|
124
|
+
if (!ids.length) {
|
|
125
|
+
return { processedRows: 0, chunksExecuted: 0, returning: [] };
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const {
|
|
129
|
+
chunkSize = DEFAULT_BULK_UPDATE_WHERE_CHUNK_SIZE,
|
|
130
|
+
concurrency = 'sequential',
|
|
131
|
+
transactional = true,
|
|
132
|
+
timing = false,
|
|
133
|
+
onChunkComplete,
|
|
134
|
+
by,
|
|
135
|
+
where: extraWhere,
|
|
136
|
+
returning,
|
|
137
|
+
} = options;
|
|
138
|
+
|
|
139
|
+
const ctx = createBulkExecutionContext(session);
|
|
140
|
+
const byColumnName = by ?? findPrimaryKey(table);
|
|
141
|
+
const byColumn = table.columns[byColumnName];
|
|
142
|
+
if (!byColumn) {
|
|
143
|
+
throw new Error(
|
|
144
|
+
`bulkUpdateWhere: column "${byColumnName}" not found in table "${table.name}"`
|
|
145
|
+
);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
const returningColumns = resolveReturningColumns(ctx, table, returning);
|
|
149
|
+
const chunks = splitIntoChunks(ids, chunkSize);
|
|
150
|
+
const totalChunks = chunks.length;
|
|
151
|
+
|
|
152
|
+
const buildTask = (chunk: ValueOperandInput[], chunkIndex: number) => async (): Promise<ChunkOutcome> => {
|
|
153
|
+
return runChunk(
|
|
154
|
+
async () => {
|
|
155
|
+
const inExpr = inList(byColumn, chunk as unknown as Parameters<typeof inList>[1]);
|
|
156
|
+
const finalWhere = extraWhere ? and(inExpr, extraWhere) : inExpr;
|
|
157
|
+
|
|
158
|
+
let builder = new UpdateQueryBuilder(table).set(set as Record<string, unknown>).where(finalWhere);
|
|
159
|
+
|
|
160
|
+
if (returningColumns?.length) {
|
|
161
|
+
builder = builder.returning(...returningColumns);
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
const compiled = builder.compile(ctx.dialect);
|
|
165
|
+
const resultSets = await executeCompiled(ctx, compiled);
|
|
166
|
+
|
|
167
|
+
return {
|
|
168
|
+
processedRows: chunk.length,
|
|
169
|
+
returning: returningColumns ? flattenQueryResults(resultSets) : [],
|
|
170
|
+
elapsedMs: 0,
|
|
171
|
+
};
|
|
172
|
+
},
|
|
173
|
+
chunkIndex,
|
|
174
|
+
totalChunks,
|
|
175
|
+
chunk.length,
|
|
176
|
+
timing,
|
|
177
|
+
onChunkComplete
|
|
178
|
+
);
|
|
179
|
+
};
|
|
180
|
+
|
|
181
|
+
const tasks = chunks.map((chunk, i) => buildTask(chunk, i));
|
|
182
|
+
|
|
183
|
+
const outcomes = await maybeTransaction(
|
|
184
|
+
session,
|
|
185
|
+
transactional,
|
|
186
|
+
() => runWithConcurrency(tasks, concurrency)
|
|
187
|
+
);
|
|
188
|
+
|
|
189
|
+
return timing
|
|
190
|
+
? aggregateOutcomesWithTimings(outcomes)
|
|
191
|
+
: aggregateOutcomes(outcomes);
|
|
192
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { InsertQueryBuilder } from '../query-builder/insert.js';
|
|
2
|
+
import { findPrimaryKey } from '../query-builder/hydration-planner.js';
|
|
3
|
+
import type { TableDef } from '../schema/table.js';
|
|
4
|
+
import type { OrmSession } from '../orm/orm-session.js';
|
|
5
|
+
import type { BulkUpsertOptions, ChunkOutcome, InsertRow } from './bulk-types.js';
|
|
6
|
+
import type { ValueOperandInput } from '../core/ast/expression.js';
|
|
7
|
+
import type { ColumnNode } from '../core/ast/expression.js';
|
|
8
|
+
import { BulkBaseExecutor, type BulkExecutorOptions } from './bulk-executor.base.js';
|
|
9
|
+
import { resolveReturningColumns, flattenQueryResults, executeCompiled } from './bulk-context.js';
|
|
10
|
+
|
|
11
|
+
interface UpsertExecutorOptions extends BulkExecutorOptions {
|
|
12
|
+
conflictColumns?: string[];
|
|
13
|
+
updateColumns?: string[];
|
|
14
|
+
returning?: boolean | import('../schema/column-types.js').ColumnDef[];
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
const DEFAULT_CHUNK_SIZE = 500;
|
|
18
|
+
|
|
19
|
+
export class BulkUpsertExecutor extends BulkBaseExecutor<UpsertExecutorOptions> {
|
|
20
|
+
private readonly conflictTargetNodes: ColumnNode[];
|
|
21
|
+
private readonly updateColumns: string[];
|
|
22
|
+
|
|
23
|
+
constructor(
|
|
24
|
+
session: OrmSession,
|
|
25
|
+
table: TableDef,
|
|
26
|
+
rows: InsertRow[],
|
|
27
|
+
options: BulkUpsertOptions = {}
|
|
28
|
+
) {
|
|
29
|
+
super(session, table, rows, { ...options, chunkSize: options.chunkSize ?? DEFAULT_CHUNK_SIZE });
|
|
30
|
+
|
|
31
|
+
const pkName = findPrimaryKey(table);
|
|
32
|
+
const conflictTargetNames = options.conflictColumns ?? [pkName];
|
|
33
|
+
this.conflictTargetNodes = conflictTargetNames.map(name => ({
|
|
34
|
+
type: 'Column' as const,
|
|
35
|
+
table: table.name,
|
|
36
|
+
name,
|
|
37
|
+
}));
|
|
38
|
+
|
|
39
|
+
const conflictSet = new Set(conflictTargetNames);
|
|
40
|
+
this.updateColumns =
|
|
41
|
+
options.updateColumns ??
|
|
42
|
+
Object.keys(rows[0] ?? {}).filter(col => !conflictSet.has(col) && col in table.columns);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
protected async executeChunk(chunk: InsertRow[], _chunkIndex: number): Promise<ChunkOutcome> {
|
|
46
|
+
const returningColumns = resolveReturningColumns(this.ctx, this.table, this.options.returning);
|
|
47
|
+
|
|
48
|
+
const set: Record<string, ValueOperandInput> = {};
|
|
49
|
+
for (const col of this.updateColumns) {
|
|
50
|
+
set[col] = { type: 'ExcludedColumn', name: col } as unknown as ValueOperandInput;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
let builder: InsertQueryBuilder<unknown>;
|
|
54
|
+
|
|
55
|
+
if (this.updateColumns.length === 0) {
|
|
56
|
+
builder = new InsertQueryBuilder(this.table).values(chunk).onConflict(this.conflictTargetNodes).doNothing();
|
|
57
|
+
} else {
|
|
58
|
+
builder = new InsertQueryBuilder(this.table)
|
|
59
|
+
.values(chunk)
|
|
60
|
+
.onConflict(this.conflictTargetNodes)
|
|
61
|
+
.doUpdate(set);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
const finalBuilder = builder as InsertQueryBuilder<unknown>;
|
|
65
|
+
|
|
66
|
+
if (returningColumns?.length) {
|
|
67
|
+
finalBuilder.returning(...returningColumns);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const compiled = finalBuilder.compile(this.ctx.dialect);
|
|
71
|
+
const resultSets = await executeCompiled(this.ctx, compiled);
|
|
72
|
+
|
|
73
|
+
return {
|
|
74
|
+
processedRows: chunk.length,
|
|
75
|
+
returning: returningColumns ? flattenQueryResults(resultSets) : [],
|
|
76
|
+
elapsedMs: 0,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export async function bulkUpsert<TTable extends TableDef>(
|
|
82
|
+
session: OrmSession,
|
|
83
|
+
table: TTable,
|
|
84
|
+
rows: InsertRow[],
|
|
85
|
+
options: BulkUpsertOptions = {}
|
|
86
|
+
): Promise<import('./bulk-types.js').BulkResult> {
|
|
87
|
+
if (!rows.length) {
|
|
88
|
+
return { processedRows: 0, chunksExecuted: 0, returning: [] };
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const executor = new BulkUpsertExecutor(session, table, rows, options);
|
|
92
|
+
return executor.execute();
|
|
93
|
+
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import type { BulkConcurrency, ChunkOutcome, ChunkCompleteInfo, BulkResult } from './bulk-types.js';
|
|
2
|
+
|
|
3
|
+
export function splitIntoChunks<T>(items: T[], size: number): T[][] {
|
|
4
|
+
if (size < 1) throw new RangeError(`chunkSize must be >= 1, got ${size}`);
|
|
5
|
+
const chunks: T[][] = [];
|
|
6
|
+
for (let i = 0; i < items.length; i += size) {
|
|
7
|
+
chunks.push(items.slice(i, i + size));
|
|
8
|
+
}
|
|
9
|
+
return chunks;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
type Task<T> = () => Promise<T>;
|
|
13
|
+
|
|
14
|
+
export async function runWithConcurrency<T>(
|
|
15
|
+
tasks: Task<T>[],
|
|
16
|
+
concurrency: BulkConcurrency
|
|
17
|
+
): Promise<T[]> {
|
|
18
|
+
const limit = concurrency === 'sequential' ? 1 : Math.max(1, concurrency);
|
|
19
|
+
|
|
20
|
+
if (limit === 1) {
|
|
21
|
+
const results: T[] = [];
|
|
22
|
+
for (const task of tasks) {
|
|
23
|
+
results.push(await task());
|
|
24
|
+
}
|
|
25
|
+
return results;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const results: T[] = new Array(tasks.length);
|
|
29
|
+
let nextIndex = 0;
|
|
30
|
+
|
|
31
|
+
const worker = async (): Promise<void> => {
|
|
32
|
+
while (nextIndex < tasks.length) {
|
|
33
|
+
const i = nextIndex++;
|
|
34
|
+
results[i] = await tasks[i]();
|
|
35
|
+
}
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
const workers = Array.from({ length: Math.min(limit, tasks.length) }, worker);
|
|
39
|
+
await Promise.all(workers);
|
|
40
|
+
return results;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export async function runChunk<T>(
|
|
44
|
+
task: Task<T>,
|
|
45
|
+
chunkIndex: number,
|
|
46
|
+
totalChunks: number,
|
|
47
|
+
rowsInChunk: number,
|
|
48
|
+
timing: boolean,
|
|
49
|
+
onChunkComplete?: (info: ChunkCompleteInfo) => void | Promise<void>
|
|
50
|
+
): Promise<T> {
|
|
51
|
+
const start = timing || onChunkComplete ? Date.now() : 0;
|
|
52
|
+
const result = await task();
|
|
53
|
+
const elapsedMs = start ? Date.now() - start : 0;
|
|
54
|
+
|
|
55
|
+
if (onChunkComplete) {
|
|
56
|
+
await onChunkComplete({ chunkIndex, totalChunks, rowsInChunk, elapsedMs });
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
return result;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export async function maybeTransaction<T>(
|
|
63
|
+
session: unknown,
|
|
64
|
+
transactional: boolean,
|
|
65
|
+
fn: () => Promise<T>
|
|
66
|
+
): Promise<T> {
|
|
67
|
+
if (!transactional) return fn();
|
|
68
|
+
const ormSession = session as { transaction: (fn: () => Promise<T>) => Promise<T> };
|
|
69
|
+
return ormSession.transaction(fn);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export function aggregateOutcomes(outcomes: ChunkOutcome[]): BulkResult {
|
|
73
|
+
const result: BulkResult = {
|
|
74
|
+
processedRows: 0,
|
|
75
|
+
chunksExecuted: outcomes.length,
|
|
76
|
+
returning: [],
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
for (const o of outcomes) {
|
|
80
|
+
result.processedRows += o.processedRows;
|
|
81
|
+
result.returning.push(...o.returning);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
return result;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export function aggregateOutcomesWithTimings(outcomes: ChunkOutcome[]): BulkResult {
|
|
88
|
+
const result = aggregateOutcomes(outcomes);
|
|
89
|
+
result.chunkTimings = outcomes.map(o => o.elapsedMs);
|
|
90
|
+
return result;
|
|
91
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export { bulkInsert, BulkInsertExecutor } from './bulk-insert-executor.js';
|
|
2
|
+
export { bulkUpdate, bulkUpdateWhere, BulkUpdateExecutor } from './bulk-update-executor.js';
|
|
3
|
+
export { bulkDelete, bulkDeleteWhere, BulkDeleteExecutor } from './bulk-delete-executor.js';
|
|
4
|
+
export { bulkUpsert, BulkUpsertExecutor } from './bulk-upsert-executor.js';
|
|
5
|
+
|
|
6
|
+
export type {
|
|
7
|
+
BulkResult,
|
|
8
|
+
BulkBaseOptions,
|
|
9
|
+
BulkConcurrency,
|
|
10
|
+
BulkInsertOptions,
|
|
11
|
+
BulkUpsertOptions,
|
|
12
|
+
BulkUpdateOptions,
|
|
13
|
+
BulkDeleteOptions,
|
|
14
|
+
InsertRow,
|
|
15
|
+
UpdateRow,
|
|
16
|
+
ChunkCompleteInfo,
|
|
17
|
+
ChunkOutcome,
|
|
18
|
+
} from './bulk-types.js';
|
|
@@ -2,9 +2,10 @@ import { OrderingTerm, SelectQueryNode } from '../core/ast/query.js';
|
|
|
2
2
|
import {
|
|
3
3
|
ExpressionNode,
|
|
4
4
|
OperandNode,
|
|
5
|
-
BinaryExpressionNode,
|
|
6
|
-
LogicalExpressionNode,
|
|
7
|
-
|
|
5
|
+
BinaryExpressionNode,
|
|
6
|
+
LogicalExpressionNode,
|
|
7
|
+
NotExpressionNode,
|
|
8
|
+
InExpressionNode,
|
|
8
9
|
NullExpressionNode,
|
|
9
10
|
JsonPathNode,
|
|
10
11
|
ExistsExpressionNode,
|
|
@@ -216,13 +217,17 @@ export class TypeScriptGenerator implements ExpressionVisitor<string>, OperandVi
|
|
|
216
217
|
return this.printBinaryExpression(binary);
|
|
217
218
|
}
|
|
218
219
|
|
|
219
|
-
public visitLogicalExpression(logical: LogicalExpressionNode): string {
|
|
220
|
-
return this.printLogicalExpression(logical);
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
public
|
|
224
|
-
return this.
|
|
225
|
-
}
|
|
220
|
+
public visitLogicalExpression(logical: LogicalExpressionNode): string {
|
|
221
|
+
return this.printLogicalExpression(logical);
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
public visitNotExpression(notExpr: NotExpressionNode): string {
|
|
225
|
+
return this.printNotExpression(notExpr);
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
public visitNullExpression(nullExpr: NullExpressionNode): string {
|
|
229
|
+
return this.printNullExpression(nullExpr);
|
|
230
|
+
}
|
|
226
231
|
|
|
227
232
|
public visitInExpression(inExpr: InExpressionNode): string {
|
|
228
233
|
return this.printInExpression(inExpr);
|
|
@@ -301,17 +306,21 @@ export class TypeScriptGenerator implements ExpressionVisitor<string>, OperandVi
|
|
|
301
306
|
* @param logical - Logical expression node
|
|
302
307
|
* @returns TypeScript code representation
|
|
303
308
|
*/
|
|
304
|
-
private printLogicalExpression(logical: LogicalExpressionNode): string {
|
|
305
|
-
if (logical.operands.length === 0) return '';
|
|
306
|
-
const parts = logical.operands.map(op => {
|
|
307
|
-
const compiled = this.printExpression(op);
|
|
308
|
-
return op.type === 'LogicalExpression' ? `(${compiled})` : compiled;
|
|
309
|
-
});
|
|
310
|
-
return `${this.mapOp(logical.operator)}(\n ${parts.join(',\n ')}\n )`;
|
|
311
|
-
}
|
|
312
|
-
|
|
313
|
-
private
|
|
314
|
-
|
|
309
|
+
private printLogicalExpression(logical: LogicalExpressionNode): string {
|
|
310
|
+
if (logical.operands.length === 0) return '';
|
|
311
|
+
const parts = logical.operands.map(op => {
|
|
312
|
+
const compiled = this.printExpression(op);
|
|
313
|
+
return op.type === 'LogicalExpression' ? `(${compiled})` : compiled;
|
|
314
|
+
});
|
|
315
|
+
return `${this.mapOp(logical.operator)}(\n ${parts.join(',\n ')}\n )`;
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
private printNotExpression(notExpr: NotExpressionNode): string {
|
|
319
|
+
return `not(${this.printExpression(notExpr.operand)})`;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
private printArithmeticExpression(expr: ArithmeticExpressionNode): string {
|
|
323
|
+
const left = this.printOperand(expr.left);
|
|
315
324
|
const right = this.printOperand(expr.right);
|
|
316
325
|
return `${left} ${expr.operator} ${right}`;
|
|
317
326
|
}
|