@memberjunction/metadata-sync 6.1.2 → 6.2.0-edge.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +88 -25
- package/dist/config.d.ts +42 -2
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js.map +1 -1
- package/dist/index.d.ts +6 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/lib/EntityPropertyExtractor.d.ts +16 -0
- package/dist/lib/EntityPropertyExtractor.d.ts.map +1 -1
- package/dist/lib/EntityPropertyExtractor.js +34 -0
- package/dist/lib/EntityPropertyExtractor.js.map +1 -1
- package/dist/lib/FieldExternalizer.d.ts +1 -2
- package/dist/lib/FieldExternalizer.d.ts.map +1 -1
- package/dist/lib/FieldExternalizer.js.map +1 -1
- package/dist/lib/RecordProcessor.d.ts +7 -0
- package/dist/lib/RecordProcessor.d.ts.map +1 -1
- package/dist/lib/RecordProcessor.js +27 -1
- package/dist/lib/RecordProcessor.js.map +1 -1
- package/dist/lib/existing-record-files.d.ts +12 -0
- package/dist/lib/existing-record-files.d.ts.map +1 -0
- package/dist/lib/existing-record-files.js +85 -0
- package/dist/lib/existing-record-files.js.map +1 -0
- package/dist/lib/file-backup-manager.d.ts +10 -0
- package/dist/lib/file-backup-manager.d.ts.map +1 -1
- package/dist/lib/file-backup-manager.js +17 -0
- package/dist/lib/file-backup-manager.js.map +1 -1
- package/dist/lib/file-write-batch.d.ts +0 -4
- package/dist/lib/file-write-batch.d.ts.map +1 -1
- package/dist/lib/file-write-batch.js +10 -8
- package/dist/lib/file-write-batch.js.map +1 -1
- package/dist/lib/graph-provider-pool.d.ts +62 -33
- package/dist/lib/graph-provider-pool.d.ts.map +1 -1
- package/dist/lib/graph-provider-pool.js +131 -79
- package/dist/lib/graph-provider-pool.js.map +1 -1
- package/dist/lib/json-subproperty-externalization.d.ts +77 -0
- package/dist/lib/json-subproperty-externalization.d.ts.map +1 -0
- package/dist/lib/json-subproperty-externalization.js +139 -0
- package/dist/lib/json-subproperty-externalization.js.map +1 -0
- package/dist/lib/push-outcome.d.ts +66 -0
- package/dist/lib/push-outcome.d.ts.map +1 -0
- package/dist/lib/push-outcome.js +81 -0
- package/dist/lib/push-outcome.js.map +1 -0
- package/dist/lib/push-write-mode.d.ts +47 -0
- package/dist/lib/push-write-mode.d.ts.map +1 -0
- package/dist/lib/push-write-mode.js +53 -0
- package/dist/lib/push-write-mode.js.map +1 -0
- package/dist/lib/record-primary-key.d.ts +35 -0
- package/dist/lib/record-primary-key.d.ts.map +1 -0
- package/dist/lib/record-primary-key.js +44 -0
- package/dist/lib/record-primary-key.js.map +1 -0
- package/dist/lib/transaction-manager.d.ts +11 -1
- package/dist/lib/transaction-manager.d.ts.map +1 -1
- package/dist/lib/transaction-manager.js +20 -4
- package/dist/lib/transaction-manager.js.map +1 -1
- package/dist/plugins/index.d.ts +1 -0
- package/dist/plugins/index.d.ts.map +1 -1
- package/dist/plugins/index.js +43 -17
- package/dist/plugins/index.js.map +1 -1
- package/dist/services/PullService.d.ts +1 -3
- package/dist/services/PullService.d.ts.map +1 -1
- package/dist/services/PullService.js +28 -68
- package/dist/services/PullService.js.map +1 -1
- package/dist/services/PushService.d.ts +109 -0
- package/dist/services/PushService.d.ts.map +1 -1
- package/dist/services/PushService.js +631 -319
- package/dist/services/PushService.js.map +1 -1
- package/package.json +14 -14
|
@@ -15,18 +15,40 @@ import { SQLLogger } from '../lib/sql-logger.js';
|
|
|
15
15
|
import { TransactionManager } from '../lib/transaction-manager.js';
|
|
16
16
|
import { JsonWriteHelper } from '../lib/json-write-helper.js';
|
|
17
17
|
import { RecordDependencyAnalyzer, groupRecordsByGraphId } from '../lib/record-dependency-analyzer.js';
|
|
18
|
-
import { GraphProviderPool } from '../lib/graph-provider-pool.js';
|
|
18
|
+
import { GraphProviderPool, probeIndependentInstances } from '../lib/graph-provider-pool.js';
|
|
19
|
+
import { resolveDirectoryMode, graphBatchSizeFor, isolatedModeWarning, unusedBatchSizeWarning, } from '../lib/push-write-mode.js';
|
|
20
|
+
import { PushAbortedError, describeCommitFailure, describeRollbackOutcome } from '../lib/push-outcome.js';
|
|
19
21
|
import { JsonPreprocessor } from '../lib/json-preprocessor.js';
|
|
20
22
|
import { findEntityDirectories } from '../lib/provider-utils.js';
|
|
21
23
|
import { DeletionAuditor } from '../lib/deletion-auditor.js';
|
|
22
24
|
import { describeMissingEntitySubclass } from '../lib/entity-subclass-guard.js';
|
|
23
25
|
import { DeletionReportGenerator } from '../lib/deletion-report-generator.js';
|
|
24
26
|
import { resolveCollectionRelationship } from '../lib/collection-resolver.js';
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
27
|
+
function emptyPushTotals() {
|
|
28
|
+
return { created: 0, updated: 0, unchanged: 0, deleted: 0, skipped: 0, deferred: 0, errors: 0 };
|
|
29
|
+
}
|
|
30
|
+
function addPushTotals(totals, result) {
|
|
31
|
+
totals.created += result.created;
|
|
32
|
+
totals.updated += result.updated;
|
|
33
|
+
totals.unchanged += result.unchanged;
|
|
34
|
+
totals.deleted += result.deleted;
|
|
35
|
+
totals.skipped += result.skipped;
|
|
36
|
+
totals.deferred += result.deferred;
|
|
37
|
+
totals.errors += result.errors;
|
|
38
|
+
}
|
|
39
|
+
/** The status a committed write is reported with, or undefined when the result wrote nothing. */
|
|
40
|
+
function committedStatusOf(result) {
|
|
41
|
+
if (result.isDuplicate || result.isDeletedRecord) {
|
|
42
|
+
return undefined;
|
|
43
|
+
}
|
|
44
|
+
if (result.status === 'updated') {
|
|
45
|
+
return 'updated';
|
|
46
|
+
}
|
|
47
|
+
if (result.status === 'created' || result.status === 'deferred') {
|
|
48
|
+
return 'created';
|
|
49
|
+
}
|
|
50
|
+
return undefined;
|
|
51
|
+
}
|
|
30
52
|
export class PushService {
|
|
31
53
|
constructor(syncEngine, contextUser, stateManager) {
|
|
32
54
|
this.warnings = [];
|
|
@@ -35,6 +57,20 @@ export class PushService {
|
|
|
35
57
|
this.deferredFileWrites = new Map();
|
|
36
58
|
this.deferredRecords = [];
|
|
37
59
|
this.confirmedCollections = new Set();
|
|
60
|
+
/** Write mode per entity directory, resolved once at the start of the push. */
|
|
61
|
+
this.directoryModes = new Map();
|
|
62
|
+
/** The directory being processed right now, and how it writes. */
|
|
63
|
+
this.writeMode = 'shared';
|
|
64
|
+
this.graphBatchSize = 1;
|
|
65
|
+
/** Counts so far, so a failed push can still report what it did. */
|
|
66
|
+
this.runningTotals = emptyPushTotals();
|
|
67
|
+
/** Non-atomic mode: creates and updates that were committed outside the push transaction. */
|
|
68
|
+
this.committedWrites = [];
|
|
69
|
+
/** Incremental state, applied only after the push commits. Keyed by path relative to the sync root. */
|
|
70
|
+
this.pendingChecksums = new Map();
|
|
71
|
+
this.pushedEntityDirs = [];
|
|
72
|
+
/** Errors whose record was already sent to onRecordError, so a caller does not report it twice. */
|
|
73
|
+
this.reportedRecordErrors = new WeakSet();
|
|
38
74
|
this.syncEngine = syncEngine;
|
|
39
75
|
this.contextUser = contextUser;
|
|
40
76
|
this.stateManager = stateManager;
|
|
@@ -133,9 +169,15 @@ export class PushService {
|
|
|
133
169
|
if (options.include && options.exclude) {
|
|
134
170
|
throw new Error('Cannot specify both --include and --exclude options. Please use one or the other.');
|
|
135
171
|
}
|
|
136
|
-
// Reset
|
|
172
|
+
// Reset per-push tracking
|
|
137
173
|
this.deferredFileWrites.clear();
|
|
138
174
|
this.deferredRecords = [];
|
|
175
|
+
this.committedWrites = [];
|
|
176
|
+
this.pendingChecksums.clear();
|
|
177
|
+
this.pushedEntityDirs = [];
|
|
178
|
+
this.directoryModes.clear();
|
|
179
|
+
this.runningTotals = emptyPushTotals();
|
|
180
|
+
this.sqlLogFilePath = undefined;
|
|
139
181
|
const fileBackupManager = new FileBackupManager();
|
|
140
182
|
// Load sync config for SQL logging settings and autoCreateMissingRecords flag
|
|
141
183
|
// If dir option is specified, load from that directory, otherwise use original CWD
|
|
@@ -156,54 +198,15 @@ export class PushService {
|
|
|
156
198
|
callbacks?.onLog?.(`SQL logging config: ${JSON.stringify(this.syncConfig?.sqlLogging)}`);
|
|
157
199
|
}
|
|
158
200
|
const sqlLogger = new SQLLogger(this.syncConfig);
|
|
159
|
-
|
|
201
|
+
// The push transaction lives on the same provider every atomic save runs on.
|
|
202
|
+
const transactionManager = new TransactionManager(sqlLogger, this.hostProvider());
|
|
160
203
|
if (options.verbose) {
|
|
161
204
|
callbacks?.onLog?.(`SQLLogger enabled status: ${sqlLogger.enabled}`);
|
|
162
205
|
}
|
|
163
206
|
// Setup SQL logging session with the provider if enabled
|
|
164
207
|
let sqlLoggingSession = null;
|
|
165
208
|
try {
|
|
166
|
-
|
|
167
|
-
if (sqlLogger.enabled && !options.dryRun) {
|
|
168
|
-
const provider = Metadata.Provider; // global-provider-ok: metadata sync operates on the configured provider only
|
|
169
|
-
if (options.verbose) {
|
|
170
|
-
callbacks?.onLog?.(`SQL logging enabled: ${sqlLogger.enabled}`);
|
|
171
|
-
callbacks?.onLog?.(`Provider type: ${provider?.constructor?.name || 'Unknown'}`);
|
|
172
|
-
callbacks?.onLog?.(`Has CreateSqlLogger: ${typeof provider?.CreateSqlLogger === 'function'}`);
|
|
173
|
-
}
|
|
174
|
-
if (provider && typeof provider.CreateSqlLogger === 'function') {
|
|
175
|
-
// Generate filename with timestamp
|
|
176
|
-
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
|
177
|
-
const filename = this.syncConfig?.sqlLogging?.formatAsMigration
|
|
178
|
-
? `MetadataSync_Push_${timestamp}.sql`
|
|
179
|
-
: `push_${timestamp}.sql`;
|
|
180
|
-
// Use .sql-log-push directory in the config directory (where sync was initiated)
|
|
181
|
-
const outputDir = path.join(configDir, this.syncConfig?.sqlLogging?.outputDirectory || './sql-log-push');
|
|
182
|
-
const filepath = path.join(outputDir, filename);
|
|
183
|
-
// Ensure the directory exists
|
|
184
|
-
await fs.ensureDir(path.dirname(filepath));
|
|
185
|
-
// Create the SQL logging session
|
|
186
|
-
sqlLoggingSession = await provider.CreateSqlLogger(filepath, {
|
|
187
|
-
formatAsMigration: this.syncConfig?.sqlLogging?.formatAsMigration || false,
|
|
188
|
-
description: 'MetadataSync push operation',
|
|
189
|
-
statementTypes: "mutations",
|
|
190
|
-
prettyPrint: true,
|
|
191
|
-
// batchSeparator is intentionally omitted — CreateSqlLogger injects the platform-appropriate
|
|
192
|
-
// separator automatically (GO for SQL Server, nothing for PostgreSQL) via PlatformBatchSeparator.
|
|
193
|
-
filterPatterns: this.syncConfig?.sqlLogging?.filterPatterns,
|
|
194
|
-
filterType: this.syncConfig?.sqlLogging?.filterType,
|
|
195
|
-
verboseOutput: this.syncConfig?.sqlLogging?.verboseOutput || false,
|
|
196
|
-
});
|
|
197
|
-
if (options.verbose) {
|
|
198
|
-
callbacks?.onLog?.(`📝 SQL logging enabled: ${filepath}`);
|
|
199
|
-
}
|
|
200
|
-
}
|
|
201
|
-
else {
|
|
202
|
-
if (options.verbose) {
|
|
203
|
-
callbacks?.onWarn?.('SQL logging requested but provider does not support it');
|
|
204
|
-
}
|
|
205
|
-
}
|
|
206
|
-
}
|
|
209
|
+
sqlLoggingSession = await this.startSqlLogging(sqlLogger, options, configDir, callbacks);
|
|
207
210
|
// Find entity directories to process
|
|
208
211
|
// Note: If options.dir is specified, configDir already points to that directory
|
|
209
212
|
// So we don't need to pass it as specificDir
|
|
@@ -211,32 +214,8 @@ export class PushService {
|
|
|
211
214
|
if (entityDirs.length === 0) {
|
|
212
215
|
throw new Error('No entity directories found');
|
|
213
216
|
}
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
// Metadata.Provider — `mj sync` is single-process so they resolve to the
|
|
217
|
-
// same instance today, but routing through SyncEngine keeps the wiring
|
|
218
|
-
// self-consistent and makes future provider plumbing trivial.
|
|
219
|
-
// Preload is internal plumbing — emit its progress only in verbose mode so a
|
|
220
|
-
// normal run jumps straight from validation to per-directory results.
|
|
221
|
-
if (options.verbose) {
|
|
222
|
-
callbacks?.onLog?.('⚡ Preloading metadata and caching files...');
|
|
223
|
-
}
|
|
224
|
-
this.syncMetadataEngine.setEntityDirs(entityDirs);
|
|
225
|
-
await this.syncMetadataEngine.Config(true, this.contextUser, this.syncEngine.getProvider());
|
|
226
|
-
for (const warning of this.syncMetadataEngine.drainWarnings()) {
|
|
227
|
-
callbacks?.onWarn?.(` ⚠️ ${warning}`);
|
|
228
|
-
}
|
|
229
|
-
if (options.verbose) {
|
|
230
|
-
const delegations = this.syncMetadataEngine.getDelegationSummary();
|
|
231
|
-
if (delegations.length > 0) {
|
|
232
|
-
const donorCount = new Set(delegations.map(d => d.engineClassName)).size;
|
|
233
|
-
callbacks?.onLog?.(` ↪ Reused in-memory caches for ${delegations.length} ${delegations.length === 1 ? 'entity' : 'entities'} already loaded by ${donorCount} ${donorCount === 1 ? 'engine' : 'engines'}`);
|
|
234
|
-
for (const d of delegations.sort((a, b) => a.entityName.localeCompare(b.entityName))) {
|
|
235
|
-
callbacks?.onLog?.(` • ${d.entityName} ← ${d.engineClassName}`);
|
|
236
|
-
}
|
|
237
|
-
}
|
|
238
|
-
callbacks?.onLog?.('✓ Preload completed successfully\n');
|
|
239
|
-
}
|
|
217
|
+
await this.applyWritePlan(entityDirs, options, callbacks);
|
|
218
|
+
await this.preloadMetadata(entityDirs, options, callbacks);
|
|
240
219
|
if (options.verbose) {
|
|
241
220
|
callbacks?.onLog?.(`Found ${entityDirs.length} entity ${entityDirs.length === 1 ? 'directory' : 'directories'} to process`);
|
|
242
221
|
}
|
|
@@ -247,14 +226,6 @@ export class PushService {
|
|
|
247
226
|
callbacks?.onLog?.('📁 File backup manager initialized');
|
|
248
227
|
}
|
|
249
228
|
}
|
|
250
|
-
// Process each entity directory
|
|
251
|
-
let totalCreated = 0;
|
|
252
|
-
let totalUpdated = 0;
|
|
253
|
-
let totalUnchanged = 0;
|
|
254
|
-
let totalDeleted = 0;
|
|
255
|
-
let totalSkipped = 0;
|
|
256
|
-
let totalDeferred = 0;
|
|
257
|
-
let totalErrors = 0;
|
|
258
229
|
// PHASE 0: Audit all deletions across all entities (if any exist)
|
|
259
230
|
let deletionAudit = null;
|
|
260
231
|
try {
|
|
@@ -268,24 +239,7 @@ export class PushService {
|
|
|
268
239
|
if (!options.dryRun && deletionAudit) {
|
|
269
240
|
const shouldProceed = await this.promptForConfirmation(deletionAudit, callbacks);
|
|
270
241
|
if (!shouldProceed) {
|
|
271
|
-
|
|
272
|
-
// Clean up SQL logging session and file if it was created
|
|
273
|
-
if (sqlLoggingSession) {
|
|
274
|
-
const sqlLogPath = sqlLoggingSession.filePath;
|
|
275
|
-
try {
|
|
276
|
-
await sqlLoggingSession.dispose();
|
|
277
|
-
// Delete the empty SQL log file since no operations occurred
|
|
278
|
-
if (await fs.pathExists(sqlLogPath)) {
|
|
279
|
-
await fs.remove(sqlLogPath);
|
|
280
|
-
if (options.verbose) {
|
|
281
|
-
callbacks?.onLog?.(`🗑️ Removed empty SQL log file: ${sqlLogPath}`);
|
|
282
|
-
}
|
|
283
|
-
}
|
|
284
|
-
}
|
|
285
|
-
catch (cleanupError) {
|
|
286
|
-
callbacks?.onWarn?.(`Failed to clean up SQL logging session: ${cleanupError}`);
|
|
287
|
-
}
|
|
288
|
-
}
|
|
242
|
+
await this.cancelPush(sqlLoggingSession, options, callbacks);
|
|
289
243
|
return {
|
|
290
244
|
created: 0,
|
|
291
245
|
updated: 0,
|
|
@@ -299,128 +253,14 @@ export class PushService {
|
|
|
299
253
|
};
|
|
300
254
|
}
|
|
301
255
|
}
|
|
302
|
-
//
|
|
303
|
-
//
|
|
304
|
-
//
|
|
256
|
+
// One host transaction wraps the whole push. In atomic mode (the default) Phase 1 saves run
|
|
257
|
+
// on the host too, one graph at a time. A directory using isolated transactions commits on
|
|
258
|
+
// independent instances as it goes (see lib/push-write-mode.ts).
|
|
305
259
|
if (!options.dryRun) {
|
|
306
260
|
await transactionManager.beginTransaction();
|
|
307
261
|
}
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
if (options.verbose) {
|
|
311
|
-
callbacks?.onLog?.('📝 Processing creates and updates...\n');
|
|
312
|
-
}
|
|
313
|
-
for (const [dirIdx, entityDir] of entityDirs.entries()) {
|
|
314
|
-
// "X of N" position prefix — only when there's more than one directory, so a
|
|
315
|
-
// single-directory push stays uncluttered.
|
|
316
|
-
const progressPrefix = entityDirs.length > 1 ? `[${dirIdx + 1}/${entityDirs.length}] ` : '';
|
|
317
|
-
const entityConfig = await loadEntityConfig(entityDir);
|
|
318
|
-
if (!entityConfig) {
|
|
319
|
-
const warning = `Skipping ${entityDir} - no valid entity configuration`;
|
|
320
|
-
this.warnings.push(warning);
|
|
321
|
-
callbacks?.onWarn?.(warning);
|
|
322
|
-
totalSkipped++; // Count skipped directories
|
|
323
|
-
continue;
|
|
324
|
-
}
|
|
325
|
-
// Show folder with spinner at start. The folder header is redundant in a
|
|
326
|
-
// normal run (the per-directory result line below names the directory), so
|
|
327
|
-
// it's verbose-only; the live spinner still shows "[X/N] Processing <dir>…".
|
|
328
|
-
const dirName = path.relative(process.cwd(), entityDir) || '.';
|
|
329
|
-
if (options.verbose) {
|
|
330
|
-
callbacks?.onLog?.(`\n📁 ${dirName}:`);
|
|
331
|
-
}
|
|
332
|
-
// Use onProgress for animated spinner if available
|
|
333
|
-
if (callbacks?.onProgress) {
|
|
334
|
-
callbacks.onProgress(`${progressPrefix}Processing ${dirName}...`);
|
|
335
|
-
}
|
|
336
|
-
else {
|
|
337
|
-
callbacks?.onLog?.(` ⏳ Processing...`);
|
|
338
|
-
}
|
|
339
|
-
if (options.verbose && callbacks?.onLog) {
|
|
340
|
-
callbacks.onLog(`Processing ${entityConfig.entity} in ${entityDir}`);
|
|
341
|
-
}
|
|
342
|
-
const result = await this.processEntityDirectory(entityDir, entityConfig, options, fileBackupManager, callbacks, configDir);
|
|
343
|
-
// Per-directory result: one compact line (always), naming the directory and
|
|
344
|
-
// its changes — or "no changes" for a clean dir. The detailed per-status
|
|
345
|
-
// breakdown is verbose-only since the final summary box already aggregates it.
|
|
346
|
-
const dirTotal = result.created + result.updated + result.unchanged + result.deleted + result.skipped;
|
|
347
|
-
const { text: dirSummary, changed: dirChanged } = this.formatDirectorySummary(progressPrefix, dirName, result, dirTotal);
|
|
348
|
-
if (callbacks?.onProgress && callbacks?.onSuccess) {
|
|
349
|
-
callbacks.onSuccess(dirSummary, dirChanged);
|
|
350
|
-
}
|
|
351
|
-
else {
|
|
352
|
-
callbacks?.onLog?.(` ${dirSummary}`);
|
|
353
|
-
}
|
|
354
|
-
if (options.verbose && (dirTotal > 0 || result.errors > 0)) {
|
|
355
|
-
callbacks?.onLog?.(` Total processed: ${dirTotal} records`);
|
|
356
|
-
if (result.created > 0) {
|
|
357
|
-
callbacks?.onLog?.(` ✓ Created: ${result.created}`);
|
|
358
|
-
}
|
|
359
|
-
if (result.updated > 0) {
|
|
360
|
-
callbacks?.onLog?.(` ✓ Updated: ${result.updated}`);
|
|
361
|
-
}
|
|
362
|
-
if (result.deleted > 0) {
|
|
363
|
-
callbacks?.onLog?.(` ✓ Deleted: ${result.deleted}`);
|
|
364
|
-
}
|
|
365
|
-
if (result.deferred > 0) {
|
|
366
|
-
callbacks?.onLog?.(` ⏳ Deferred: ${result.deferred}`);
|
|
367
|
-
}
|
|
368
|
-
if (result.unchanged > 0) {
|
|
369
|
-
callbacks?.onLog?.(` - Unchanged: ${result.unchanged}`);
|
|
370
|
-
}
|
|
371
|
-
if (result.skipped > 0) {
|
|
372
|
-
callbacks?.onLog?.(` - Skipped: ${result.skipped}`);
|
|
373
|
-
}
|
|
374
|
-
if (result.errors > 0) {
|
|
375
|
-
callbacks?.onLog?.(` ✗ Errors: ${result.errors}`);
|
|
376
|
-
}
|
|
377
|
-
}
|
|
378
|
-
totalCreated += result.created;
|
|
379
|
-
totalUpdated += result.updated;
|
|
380
|
-
totalUnchanged += result.unchanged;
|
|
381
|
-
totalDeleted += result.deleted;
|
|
382
|
-
totalSkipped += result.skipped;
|
|
383
|
-
totalDeferred += result.deferred;
|
|
384
|
-
totalErrors += result.errors;
|
|
385
|
-
}
|
|
386
|
-
// PHASE 2: Process deletions in reverse dependency order (if any exist)
|
|
387
|
-
if (deletionAudit && totalErrors === 0) {
|
|
388
|
-
const deletionResult = await this.processDeletionsFromAudit(deletionAudit, options, callbacks);
|
|
389
|
-
totalDeleted += deletionResult.deleted;
|
|
390
|
-
totalErrors += deletionResult.errors;
|
|
391
|
-
}
|
|
392
|
-
// PHASE 2.5: Process deferred records (for circular dependencies)
|
|
393
|
-
if (this.deferredRecords.length > 0 && totalErrors === 0) {
|
|
394
|
-
const deferredResult = await this.processDeferredRecords(options, callbacks);
|
|
395
|
-
totalCreated += deferredResult.created;
|
|
396
|
-
totalUpdated += deferredResult.updated;
|
|
397
|
-
totalErrors += deferredResult.errors;
|
|
398
|
-
}
|
|
399
|
-
// Commit transaction if successful
|
|
400
|
-
if (!options.dryRun && totalErrors === 0) {
|
|
401
|
-
await transactionManager.commitTransaction();
|
|
402
|
-
}
|
|
403
|
-
// PHASE 3: Write deferred files with updated deletion timestamps
|
|
404
|
-
if (!options.dryRun && totalErrors === 0 && this.deferredFileWrites.size > 0) {
|
|
405
|
-
await this.writeDeferredFiles(options, callbacks);
|
|
406
|
-
}
|
|
407
|
-
}
|
|
408
|
-
catch (error) {
|
|
409
|
-
// Rollback transaction on error.
|
|
410
|
-
if (!options.dryRun) {
|
|
411
|
-
callbacks?.onLog?.('\n⚠️ Rolling back database transaction due to error...');
|
|
412
|
-
const rolledBack = await transactionManager.rollbackTransaction();
|
|
413
|
-
if (rolledBack) {
|
|
414
|
-
callbacks?.onLog?.('✓ Database transaction rolled back successfully\n');
|
|
415
|
-
}
|
|
416
|
-
else {
|
|
417
|
-
callbacks?.onLog?.('❌ Database transaction rollback failed\n');
|
|
418
|
-
}
|
|
419
|
-
}
|
|
420
|
-
throw error;
|
|
421
|
-
}
|
|
422
|
-
// Commit file backups if successful and not in dry-run mode
|
|
423
|
-
if (!options.dryRun && totalErrors === 0) {
|
|
262
|
+
const totals = await this.runPushInTransaction({ entityDirs, deletionAudit, options, fileBackupManager, callbacks, configDir }, transactionManager);
|
|
263
|
+
if (!options.dryRun) {
|
|
424
264
|
await fileBackupManager.cleanup();
|
|
425
265
|
if (options.verbose) {
|
|
426
266
|
callbacks?.onLog?.('✅ File backups committed');
|
|
@@ -443,13 +283,7 @@ export class PushService {
|
|
|
443
283
|
}
|
|
444
284
|
}
|
|
445
285
|
return {
|
|
446
|
-
|
|
447
|
-
updated: totalUpdated,
|
|
448
|
-
unchanged: totalUnchanged,
|
|
449
|
-
deleted: totalDeleted,
|
|
450
|
-
skipped: totalSkipped,
|
|
451
|
-
deferred: totalDeferred,
|
|
452
|
-
errors: totalErrors,
|
|
286
|
+
...totals,
|
|
453
287
|
warnings: this.warnings,
|
|
454
288
|
sqlLogPath,
|
|
455
289
|
changeLog: this.changeDetails
|
|
@@ -478,6 +312,381 @@ export class PushService {
|
|
|
478
312
|
throw error;
|
|
479
313
|
}
|
|
480
314
|
}
|
|
315
|
+
/** Open the SQL logging session when the config asks for one and this is not a dry run. */
|
|
316
|
+
async startSqlLogging(sqlLogger, options, configDir, callbacks) {
|
|
317
|
+
let sqlLoggingSession = null;
|
|
318
|
+
if (sqlLogger.enabled && !options.dryRun) {
|
|
319
|
+
const provider = Metadata.Provider; // global-provider-ok: metadata sync operates on the configured provider only
|
|
320
|
+
if (options.verbose) {
|
|
321
|
+
callbacks?.onLog?.(`SQL logging enabled: ${sqlLogger.enabled}`);
|
|
322
|
+
callbacks?.onLog?.(`Provider type: ${provider?.constructor?.name || 'Unknown'}`);
|
|
323
|
+
callbacks?.onLog?.(`Has CreateSqlLogger: ${typeof provider?.CreateSqlLogger === 'function'}`);
|
|
324
|
+
}
|
|
325
|
+
if (provider && typeof provider.CreateSqlLogger === 'function') {
|
|
326
|
+
// Generate filename with timestamp
|
|
327
|
+
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
|
328
|
+
const filename = this.syncConfig?.sqlLogging?.formatAsMigration
|
|
329
|
+
? `MetadataSync_Push_${timestamp}.sql`
|
|
330
|
+
: `push_${timestamp}.sql`;
|
|
331
|
+
// Use .sql-log-push directory in the config directory (where sync was initiated)
|
|
332
|
+
const outputDir = path.join(configDir, this.syncConfig?.sqlLogging?.outputDirectory || './sql-log-push');
|
|
333
|
+
const filepath = path.join(outputDir, filename);
|
|
334
|
+
// Ensure the directory exists
|
|
335
|
+
await fs.ensureDir(path.dirname(filepath));
|
|
336
|
+
// Create the SQL logging session
|
|
337
|
+
this.sqlLogFilePath = filepath;
|
|
338
|
+
sqlLoggingSession = await provider.CreateSqlLogger(filepath, {
|
|
339
|
+
formatAsMigration: this.syncConfig?.sqlLogging?.formatAsMigration || false,
|
|
340
|
+
description: 'MetadataSync push operation',
|
|
341
|
+
statementTypes: "mutations",
|
|
342
|
+
prettyPrint: true,
|
|
343
|
+
// batchSeparator is intentionally omitted — CreateSqlLogger injects the platform-appropriate
|
|
344
|
+
// separator automatically (GO for SQL Server, nothing for PostgreSQL) via PlatformBatchSeparator.
|
|
345
|
+
filterPatterns: this.syncConfig?.sqlLogging?.filterPatterns,
|
|
346
|
+
filterType: this.syncConfig?.sqlLogging?.filterType,
|
|
347
|
+
verboseOutput: this.syncConfig?.sqlLogging?.verboseOutput || false,
|
|
348
|
+
});
|
|
349
|
+
if (options.verbose) {
|
|
350
|
+
callbacks?.onLog?.(`📝 SQL logging enabled: ${filepath}`);
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
else {
|
|
354
|
+
if (options.verbose) {
|
|
355
|
+
callbacks?.onWarn?.('SQL logging requested but provider does not support it');
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
return sqlLoggingSession;
|
|
360
|
+
}
|
|
361
|
+
/**
|
|
362
|
+
* Preload entity metadata and cache the files on disk.
|
|
363
|
+
*
|
|
364
|
+
* The SyncEngine's provider is passed explicitly rather than reaching for Metadata.Provider —
|
|
365
|
+
* `mj sync` is single-process so they resolve to the same instance today, but routing through
|
|
366
|
+
* SyncEngine keeps the wiring self-consistent.
|
|
367
|
+
*/
|
|
368
|
+
async preloadMetadata(entityDirs, options, callbacks) {
|
|
369
|
+
// We pass the SyncEngine's provider explicitly rather than reaching for
|
|
370
|
+
// Metadata.Provider — `mj sync` is single-process so they resolve to the
|
|
371
|
+
// same instance today, but routing through SyncEngine keeps the wiring
|
|
372
|
+
// self-consistent and makes future provider plumbing trivial.
|
|
373
|
+
// Preload is internal plumbing — emit its progress only in verbose mode so a
|
|
374
|
+
// normal run jumps straight from validation to per-directory results.
|
|
375
|
+
if (options.verbose) {
|
|
376
|
+
callbacks?.onLog?.('⚡ Preloading metadata and caching files...');
|
|
377
|
+
}
|
|
378
|
+
this.syncMetadataEngine.setEntityDirs(entityDirs);
|
|
379
|
+
await this.syncMetadataEngine.Config(true, this.contextUser, this.syncEngine.getProvider());
|
|
380
|
+
for (const warning of this.syncMetadataEngine.drainWarnings()) {
|
|
381
|
+
callbacks?.onWarn?.(` ⚠️ ${warning}`);
|
|
382
|
+
}
|
|
383
|
+
if (options.verbose) {
|
|
384
|
+
const delegations = this.syncMetadataEngine.getDelegationSummary();
|
|
385
|
+
if (delegations.length > 0) {
|
|
386
|
+
const donorCount = new Set(delegations.map(d => d.engineClassName)).size;
|
|
387
|
+
callbacks?.onLog?.(` ↪ Reused in-memory caches for ${delegations.length} ${delegations.length === 1 ? 'entity' : 'entities'} already loaded by ${donorCount} ${donorCount === 1 ? 'engine' : 'engines'}`);
|
|
388
|
+
for (const d of delegations.sort((a, b) => a.entityName.localeCompare(b.entityName))) {
|
|
389
|
+
callbacks?.onLog?.(` • ${d.entityName} ← ${d.engineClassName}`);
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
callbacks?.onLog?.('✓ Preload completed successfully\n');
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
/** The user declined the deletion prompt: drop an empty SQL log and say nothing happened. */
|
|
396
|
+
async cancelPush(sqlLoggingSession, options, callbacks) {
|
|
397
|
+
callbacks?.onLog?.('\n❌ Push operation cancelled by user.\n');
|
|
398
|
+
// Clean up SQL logging session and file if it was created
|
|
399
|
+
if (sqlLoggingSession) {
|
|
400
|
+
const sqlLogPath = sqlLoggingSession.filePath;
|
|
401
|
+
try {
|
|
402
|
+
await sqlLoggingSession.dispose();
|
|
403
|
+
// Delete the empty SQL log file since no operations occurred
|
|
404
|
+
if (await fs.pathExists(sqlLogPath)) {
|
|
405
|
+
await fs.remove(sqlLogPath);
|
|
406
|
+
if (options.verbose) {
|
|
407
|
+
callbacks?.onLog?.(`🗑️ Removed empty SQL log file: ${sqlLogPath}`);
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
catch (cleanupError) {
|
|
412
|
+
callbacks?.onWarn?.(`Failed to clean up SQL logging session: ${cleanupError}`);
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
/**
|
|
417
|
+
* Decide, once and up front, how each entity directory writes: shared by default, isolated where
|
|
418
|
+
* the entity (or the root, or the CLI flag) asks for it. Deciding it here means a push can never
|
|
419
|
+
* discover mid-file that it must change topology, which is the mixed-provider deadlock.
|
|
420
|
+
*/
|
|
421
|
+
async applyWritePlan(entityDirs, options, callbacks) {
|
|
422
|
+
const isolated = [];
|
|
423
|
+
for (const entityDir of entityDirs) {
|
|
424
|
+
const entityConfig = await loadEntityConfig(entityDir);
|
|
425
|
+
const resolved = resolveDirectoryMode({
|
|
426
|
+
isolatedFlag: options.isolatedTransactions,
|
|
427
|
+
entityIsolated: entityConfig?.push?.isolatedTransactions,
|
|
428
|
+
rootIsolated: this.syncConfig?.push?.isolatedTransactions,
|
|
429
|
+
});
|
|
430
|
+
this.directoryModes.set(entityDir, resolved.mode);
|
|
431
|
+
if (resolved.mode === 'isolated') {
|
|
432
|
+
isolated.push(path.relative(process.cwd(), entityDir) || entityDir);
|
|
433
|
+
}
|
|
434
|
+
if (options.verbose) {
|
|
435
|
+
callbacks?.onLog?.(` ${path.relative(process.cwd(), entityDir) || entityDir}: ${resolved.mode} (from ${resolved.source})`);
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
if (isolated.length > 0) {
|
|
439
|
+
await this.confirmIsolatedTransactions(isolated, options, callbacks);
|
|
440
|
+
}
|
|
441
|
+
else if (options.parallelBatchSize !== undefined && options.parallelBatchSize !== 1) {
|
|
442
|
+
this.addWarning(unusedBatchSizeWarning(options.parallelBatchSize), callbacks);
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
/**
|
|
446
|
+
* Isolation needs independent provider instances. Without them every directory runs shared,
|
|
447
|
+
* because the alternative — some graphs on the host, some on their own connection — is the
|
|
448
|
+
* deadlock the graph pool exists to prevent.
|
|
449
|
+
*/
|
|
450
|
+
async confirmIsolatedTransactions(isolated, options, callbacks) {
|
|
451
|
+
const reason = await probeIndependentInstances(this.hostProvider());
|
|
452
|
+
if (reason) {
|
|
453
|
+
this.addWarning(`Independent provider instances are not available (${reason}), so every directory runs in the shared ` +
|
|
454
|
+
`push transaction: one transaction, one graph at a time.`, callbacks);
|
|
455
|
+
for (const dir of this.directoryModes.keys()) {
|
|
456
|
+
this.directoryModes.set(dir, 'shared');
|
|
457
|
+
}
|
|
458
|
+
return;
|
|
459
|
+
}
|
|
460
|
+
if (!options.dryRun) {
|
|
461
|
+
this.addWarning(isolatedModeWarning(isolated, graphBatchSizeFor('isolated', options.parallelBatchSize)), callbacks);
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
/** Adopt one directory's mode for the work about to run in it. */
|
|
465
|
+
useDirectoryMode(entityDir, options) {
|
|
466
|
+
this.writeMode = this.directoryModes.get(entityDir) ?? 'shared';
|
|
467
|
+
this.graphBatchSize = graphBatchSizeFor(this.writeMode, options.parallelBatchSize);
|
|
468
|
+
}
|
|
469
|
+
addWarning(message, callbacks) {
|
|
470
|
+
this.warnings.push(message);
|
|
471
|
+
callbacks?.onWarn?.(`⚠️ ${message}`);
|
|
472
|
+
}
|
|
473
|
+
/** The process-wide provider that owns the push transaction. */
|
|
474
|
+
hostProvider() {
|
|
475
|
+
return Metadata.Provider; // global-provider-ok: mj sync is single-process; this provider owns the push transaction
|
|
476
|
+
}
|
|
477
|
+
/**
|
|
478
|
+
* Run every phase inside the push transaction, commit it, then do the post-commit work.
|
|
479
|
+
* Never returns or throws with the transaction still open.
|
|
480
|
+
*/
|
|
481
|
+
async runPushInTransaction(run, transactionManager) {
|
|
482
|
+
let committed = false;
|
|
483
|
+
try {
|
|
484
|
+
const totals = await this.runPushPhases(run);
|
|
485
|
+
if (!run.options.dryRun) {
|
|
486
|
+
await this.commitPushTransaction(transactionManager);
|
|
487
|
+
committed = true;
|
|
488
|
+
// PHASE 3: files that contained deletions, then the incremental state
|
|
489
|
+
await this.writeDeferredFiles(run.options, run.callbacks);
|
|
490
|
+
await this.persistIncrementalState(run.configDir);
|
|
491
|
+
}
|
|
492
|
+
return totals;
|
|
493
|
+
}
|
|
494
|
+
catch (error) {
|
|
495
|
+
if (committed) {
|
|
496
|
+
throw await this.failAfterCommit(error, run);
|
|
497
|
+
}
|
|
498
|
+
throw await this.abortPush(error, transactionManager, run);
|
|
499
|
+
}
|
|
500
|
+
finally {
|
|
501
|
+
await this.ensureTransactionClosed(transactionManager, run.callbacks);
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
/** Phases 1, 2 and 2.5. Any failure throws; counted errors only survive a dry run. */
|
|
505
|
+
async runPushPhases(run) {
|
|
506
|
+
const { options, callbacks } = run;
|
|
507
|
+
if (options.verbose) {
|
|
508
|
+
callbacks?.onLog?.('📝 Processing creates and updates...\n');
|
|
509
|
+
}
|
|
510
|
+
// PHASE 1: creates and updates
|
|
511
|
+
const totals = await this.processAllEntityDirectories(run);
|
|
512
|
+
// PHASE 2: deletions in reverse dependency order
|
|
513
|
+
if (run.deletionAudit && totals.errors === 0) {
|
|
514
|
+
const deletionResult = await this.processDeletionsFromAudit(run.deletionAudit, options, callbacks);
|
|
515
|
+
totals.deleted += deletionResult.deleted;
|
|
516
|
+
totals.errors += deletionResult.errors;
|
|
517
|
+
}
|
|
518
|
+
// PHASE 2.5: deferred records (circular dependencies)
|
|
519
|
+
if (this.deferredRecords.length > 0 && totals.errors === 0) {
|
|
520
|
+
const deferredResult = await this.processDeferredRecords(options, callbacks);
|
|
521
|
+
totals.created += deferredResult.created;
|
|
522
|
+
totals.updated += deferredResult.updated;
|
|
523
|
+
totals.errors += deferredResult.errors;
|
|
524
|
+
}
|
|
525
|
+
if (!options.dryRun && totals.errors > 0) {
|
|
526
|
+
throw new Error(`${totals.errors} record error${totals.errors === 1 ? '' : 's'} occurred; the push was not committed.`);
|
|
527
|
+
}
|
|
528
|
+
return totals;
|
|
529
|
+
}
|
|
530
|
+
async commitPushTransaction(transactionManager) {
|
|
531
|
+
try {
|
|
532
|
+
await transactionManager.commitTransaction();
|
|
533
|
+
}
|
|
534
|
+
catch (error) {
|
|
535
|
+
throw new Error(describeCommitFailure(error, this.hostProvider()?.PlatformKey), { cause: error });
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
/** Roll back, say truthfully what is left in the database, and wrap the failure. */
|
|
539
|
+
async abortPush(error, transactionManager, run) {
|
|
540
|
+
const { options, callbacks } = run;
|
|
541
|
+
let rolledBack = true;
|
|
542
|
+
if (!options.dryRun) {
|
|
543
|
+
callbacks?.onWarn?.('\n⚠️ Rolling back database transaction due to error...');
|
|
544
|
+
rolledBack = await transactionManager.rollbackTransaction();
|
|
545
|
+
await this.writeFilesWithCommittedRecords(run);
|
|
546
|
+
for (const line of describeRollbackOutcome(rolledBack, this.committedWrites, configManager.getOriginalCwd())) {
|
|
547
|
+
callbacks?.onWarn?.(line);
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
return new PushAbortedError({
|
|
551
|
+
modes: [...new Set(this.directoryModes.values())],
|
|
552
|
+
rolledBack,
|
|
553
|
+
committedWrites: [...this.committedWrites],
|
|
554
|
+
totals: { ...this.runningTotals },
|
|
555
|
+
sqlLogPath: this.sqlLogFilePath,
|
|
556
|
+
cause: error,
|
|
557
|
+
});
|
|
558
|
+
}
|
|
559
|
+
/**
|
|
560
|
+
* A file whose write was deferred to Phase 3 — it contains deletions — never reaches that phase
|
|
561
|
+
* when the push fails. In an isolated directory its creates are committed all the same, so write
|
|
562
|
+
* it now and keep it: otherwise the primary keys those rows were given exist only in the
|
|
563
|
+
* database, and the next push creates them a second time.
|
|
564
|
+
*/
|
|
565
|
+
async writeFilesWithCommittedRecords(run) {
|
|
566
|
+
const committedFiles = new Set(this.committedWrites.map((w) => w.filePath));
|
|
567
|
+
for (const deferred of this.deferredFileWrites.values()) {
|
|
568
|
+
if (!committedFiles.has(deferred.filePath)) {
|
|
569
|
+
continue;
|
|
570
|
+
}
|
|
571
|
+
try {
|
|
572
|
+
const payload = deferred.isArray ? deferred.records : deferred.records[0];
|
|
573
|
+
await JsonWriteHelper.writeOrderedRecordData(deferred.filePath, payload);
|
|
574
|
+
this.syncMetadataEngine.invalidateCachedFile(deferred.filePath);
|
|
575
|
+
run.fileBackupManager.releaseBackup(deferred.filePath);
|
|
576
|
+
run.callbacks?.onWarn?.(` kept ${path.relative(configManager.getOriginalCwd(), deferred.filePath) || deferred.filePath}: ` +
|
|
577
|
+
`it holds records that are committed`);
|
|
578
|
+
}
|
|
579
|
+
catch (writeError) {
|
|
580
|
+
run.callbacks?.onWarn?.(`Failed to write ${deferred.filePath} after a partial push: ${writeError instanceof Error ? writeError.message : String(writeError)}`);
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
/**
|
|
585
|
+
* The database already committed, so the metadata files must keep what they were written with.
|
|
586
|
+
* Drop the file backups so the caller's error path does not restore stale files.
|
|
587
|
+
*/
|
|
588
|
+
async failAfterCommit(error, run) {
|
|
589
|
+
await run.fileBackupManager.cleanup();
|
|
590
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
591
|
+
run.callbacks?.onWarn?.(`⚠️ The database changes were committed, but a step after the commit failed: ${message}. ` +
|
|
592
|
+
`Some metadata files or the incremental state may not match the database; run the push again to bring them in line.`);
|
|
593
|
+
return error;
|
|
594
|
+
}
|
|
595
|
+
async ensureTransactionClosed(transactionManager, callbacks) {
|
|
596
|
+
if (!transactionManager.isInTransaction) {
|
|
597
|
+
return;
|
|
598
|
+
}
|
|
599
|
+
const rolledBack = await transactionManager.rollbackTransaction();
|
|
600
|
+
callbacks?.onWarn?.(rolledBack
|
|
601
|
+
? '⚠️ The push transaction was still open when the push ended, so it was rolled back.'
|
|
602
|
+
: '❌ The push transaction was still open when the push ended, and rolling it back failed.');
|
|
603
|
+
}
|
|
604
|
+
/** Apply incremental checksums and push timestamps. Called only after the push commits. */
|
|
605
|
+
async persistIncrementalState(syncRootDir) {
|
|
606
|
+
if (!this.stateManager) {
|
|
607
|
+
return;
|
|
608
|
+
}
|
|
609
|
+
for (const [relativePath, checksum] of this.pendingChecksums) {
|
|
610
|
+
this.stateManager.setFileChecksum(relativePath, checksum);
|
|
611
|
+
}
|
|
612
|
+
const pushedAt = new Date().toISOString();
|
|
613
|
+
for (const relativeEntityDir of this.pushedEntityDirs) {
|
|
614
|
+
this.stateManager.setLastPushTimestamp(relativeEntityDir, pushedAt);
|
|
615
|
+
}
|
|
616
|
+
await this.stateManager.pruneStaleChecksums(syncRootDir);
|
|
617
|
+
await this.stateManager.save();
|
|
618
|
+
}
|
|
619
|
+
/** PHASE 1 over every entity directory, in order. */
|
|
620
|
+
async processAllEntityDirectories(run) {
|
|
621
|
+
const { entityDirs, options, callbacks } = run;
|
|
622
|
+
const totals = emptyPushTotals();
|
|
623
|
+
for (const [dirIdx, entityDir] of entityDirs.entries()) {
|
|
624
|
+
// "X of N" position prefix — only when there's more than one directory.
|
|
625
|
+
const progressPrefix = entityDirs.length > 1 ? `[${dirIdx + 1}/${entityDirs.length}] ` : '';
|
|
626
|
+
const entityConfig = await loadEntityConfig(entityDir);
|
|
627
|
+
if (!entityConfig) {
|
|
628
|
+
const warning = `Skipping ${entityDir} - no valid entity configuration`;
|
|
629
|
+
this.warnings.push(warning);
|
|
630
|
+
callbacks?.onWarn?.(warning);
|
|
631
|
+
totals.skipped++;
|
|
632
|
+
continue;
|
|
633
|
+
}
|
|
634
|
+
const dirName = path.relative(process.cwd(), entityDir) || '.';
|
|
635
|
+
this.useDirectoryMode(entityDir, options);
|
|
636
|
+
this.announceDirectory(dirName, progressPrefix, entityConfig.entity, entityDir, options, callbacks);
|
|
637
|
+
const result = await this.processEntityDirectory(entityDir, entityConfig, options, run.fileBackupManager, callbacks, run.configDir);
|
|
638
|
+
this.reportDirectoryResult(progressPrefix, dirName, result, options, callbacks);
|
|
639
|
+
addPushTotals(totals, result);
|
|
640
|
+
addPushTotals(this.runningTotals, result);
|
|
641
|
+
}
|
|
642
|
+
return totals;
|
|
643
|
+
}
|
|
644
|
+
announceDirectory(dirName, progressPrefix, entityName, entityDir, options, callbacks) {
|
|
645
|
+
// The folder header is verbose-only; the live spinner still shows "[X/N] Processing <dir>…".
|
|
646
|
+
if (options.verbose) {
|
|
647
|
+
callbacks?.onLog?.(`\n📁 ${dirName}:`);
|
|
648
|
+
}
|
|
649
|
+
if (callbacks?.onProgress) {
|
|
650
|
+
callbacks.onProgress(`${progressPrefix}Processing ${dirName}...`);
|
|
651
|
+
}
|
|
652
|
+
else {
|
|
653
|
+
callbacks?.onLog?.(` ⏳ Processing...`);
|
|
654
|
+
}
|
|
655
|
+
if (options.verbose) {
|
|
656
|
+
callbacks?.onLog?.(`Processing ${entityName} in ${entityDir}`);
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
/** One compact line per directory, plus the per-status breakdown in verbose mode. */
|
|
660
|
+
reportDirectoryResult(progressPrefix, dirName, result, options, callbacks) {
|
|
661
|
+
const dirTotal = result.created + result.updated + result.unchanged + result.deleted + result.skipped;
|
|
662
|
+
const { text: dirSummary, changed: dirChanged } = this.formatDirectorySummary(progressPrefix, dirName, result, dirTotal);
|
|
663
|
+
if (callbacks?.onProgress && callbacks?.onSuccess) {
|
|
664
|
+
callbacks.onSuccess(dirSummary, dirChanged);
|
|
665
|
+
}
|
|
666
|
+
else {
|
|
667
|
+
callbacks?.onLog?.(` ${dirSummary}`);
|
|
668
|
+
}
|
|
669
|
+
if (options.verbose && (dirTotal > 0 || result.errors > 0)) {
|
|
670
|
+
this.logDirectoryBreakdown(result, dirTotal, callbacks);
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
logDirectoryBreakdown(result, dirTotal, callbacks) {
|
|
674
|
+
const lines = [
|
|
675
|
+
[result.created, ` ✓ Created: ${result.created}`],
|
|
676
|
+
[result.updated, ` ✓ Updated: ${result.updated}`],
|
|
677
|
+
[result.deleted, ` ✓ Deleted: ${result.deleted}`],
|
|
678
|
+
[result.deferred, ` ⏳ Deferred: ${result.deferred}`],
|
|
679
|
+
[result.unchanged, ` - Unchanged: ${result.unchanged}`],
|
|
680
|
+
[result.skipped, ` - Skipped: ${result.skipped}`],
|
|
681
|
+
[result.errors, ` ✗ Errors: ${result.errors}`],
|
|
682
|
+
];
|
|
683
|
+
callbacks?.onLog?.(` Total processed: ${dirTotal} records`);
|
|
684
|
+
for (const [count, line] of lines) {
|
|
685
|
+
if (count > 0) {
|
|
686
|
+
callbacks?.onLog?.(line);
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
}
|
|
481
690
|
async processEntityDirectory(entityDir, entityConfig, options, fileBackupManager, callbacks, syncRootDir) {
|
|
482
691
|
let created = 0;
|
|
483
692
|
let updated = 0;
|
|
@@ -563,18 +772,9 @@ export class PushService {
|
|
|
563
772
|
if (options.verbose) {
|
|
564
773
|
callbacks?.onLog?.(` Analyzed ${analysisResult.sortedRecords.length} records (including nested)`);
|
|
565
774
|
}
|
|
566
|
-
//
|
|
567
|
-
//
|
|
568
|
-
// Map operations themselves are atomic, but we ensure records are added to
|
|
569
|
-
// the context AFTER successful save to maintain consistency.
|
|
775
|
+
// Records are added to the batch context only AFTER a successful save (applyProcessResult),
|
|
776
|
+
// so later lookups in this file see consistent state.
|
|
570
777
|
const batchContext = new BatchContextIndex();
|
|
571
|
-
// One provider per JSON-root graph (Action + nested Action Params share a
|
|
572
|
-
// connection). Parallelize sibling roots only. Drain a graph when its last
|
|
573
|
-
// level finishes, or when TransactionDepth is already 0 (Save settled).
|
|
574
|
-
// Peak live independent instances is then the current batch plus any
|
|
575
|
-
// leftover-depth graphs still spanning later levels.
|
|
576
|
-
const hostProvider = Metadata.Provider; // global-provider-ok: host provider template for GraphProviderPool cloning
|
|
577
|
-
const graphPool = new GraphProviderPool(hostProvider, (msg) => callbacks?.onLog?.(msg));
|
|
578
778
|
const applyProcessResult = (result) => {
|
|
579
779
|
if (result.batchContextEntry) {
|
|
580
780
|
batchContext.set(result.batchContextEntry.key, result.batchContextEntry.entity);
|
|
@@ -601,83 +801,27 @@ export class PushService {
|
|
|
601
801
|
deleted++;
|
|
602
802
|
else if (result.status === 'skipped')
|
|
603
803
|
skipped++;
|
|
604
|
-
else if (result.status === 'error')
|
|
605
|
-
// A non-throwing record error must not commit leftover graph depth —
|
|
606
|
-
// the previous per-record release rolled that work back.
|
|
804
|
+
else if (result.status === 'error')
|
|
607
805
|
errors++;
|
|
608
|
-
graphPool.markFailed();
|
|
609
|
-
}
|
|
610
806
|
else if (result.status === 'deferred') {
|
|
611
807
|
created++;
|
|
612
808
|
deferred++;
|
|
613
809
|
}
|
|
614
810
|
};
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
if (options.verbose && graphIds.length > 1) {
|
|
630
|
-
callbacks?.onLog?.(` Level ${levelIndex}: ${level.length} records in ${graphIds.length} graphs (parallel batch ${batchSize})`);
|
|
631
|
-
}
|
|
632
|
-
for (let i = 0; i < graphIds.length; i += batchSize) {
|
|
633
|
-
const batchIds = graphIds.slice(i, i + batchSize);
|
|
634
|
-
const batchResults = await Promise.all(batchIds.map(async (graphId) => {
|
|
635
|
-
const recs = byGraph.get(graphId);
|
|
636
|
-
const provider = await graphPool.obtain(graphId);
|
|
637
|
-
const results = [];
|
|
638
|
-
for (const flattenedRecord of recs) {
|
|
639
|
-
try {
|
|
640
|
-
const result = await this.processFlattenedRecord(flattenedRecord, entityDir, options, batchContext, callbacks, entityConfig, true, provider);
|
|
641
|
-
results.push({ success: true, result, record: flattenedRecord, graphId });
|
|
642
|
-
}
|
|
643
|
-
catch (error) {
|
|
644
|
-
graphPool.markFailed();
|
|
645
|
-
results.push({ success: false, error, record: flattenedRecord, graphId });
|
|
646
|
-
break;
|
|
647
|
-
}
|
|
648
|
-
}
|
|
649
|
-
return results;
|
|
650
|
-
}));
|
|
651
|
-
for (const graphResults of batchResults) {
|
|
652
|
-
for (const batchResult of graphResults) {
|
|
653
|
-
if (batchResult.success === false) {
|
|
654
|
-
const err = batchResult.error;
|
|
655
|
-
const rec = batchResult.record;
|
|
656
|
-
callbacks?.onLog?.(`\n❌ Processing failed for ${rec.entityName} at ${rec.path}`);
|
|
657
|
-
callbacks?.onLog?.(` ${err.message}\n`);
|
|
658
|
-
if (err.stack) {
|
|
659
|
-
callbacks?.onLog?.(` Stack: ${err.stack}\n`);
|
|
660
|
-
}
|
|
661
|
-
throw err;
|
|
662
|
-
}
|
|
663
|
-
applyProcessResult(batchResult.result);
|
|
664
|
-
}
|
|
665
|
-
}
|
|
666
|
-
const drainError = await graphPool.drainBatch(batchIds, levelIndex);
|
|
667
|
-
if (drainError)
|
|
668
|
-
throw drainError;
|
|
669
|
-
}
|
|
670
|
-
}
|
|
671
|
-
}
|
|
672
|
-
catch (e) {
|
|
673
|
-
graphPool.markFailed();
|
|
674
|
-
runError = e;
|
|
675
|
-
}
|
|
676
|
-
const settleError = await graphPool.releaseAll();
|
|
677
|
-
if (runError)
|
|
678
|
-
throw runError;
|
|
679
|
-
if (settleError)
|
|
680
|
-
throw settleError;
|
|
811
|
+
const levels = analysisResult.dependencyLevels && analysisResult.dependencyLevels.length > 0
|
|
812
|
+
? analysisResult.dependencyLevels
|
|
813
|
+
: [analysisResult.sortedRecords];
|
|
814
|
+
await this.runFileGraphs({
|
|
815
|
+
filePath,
|
|
816
|
+
entityDir,
|
|
817
|
+
entityConfig,
|
|
818
|
+
options,
|
|
819
|
+
callbacks,
|
|
820
|
+
batchContext,
|
|
821
|
+
levels,
|
|
822
|
+
applyResult: applyProcessResult,
|
|
823
|
+
errorCount: () => errors,
|
|
824
|
+
});
|
|
681
825
|
// Check if this file has any deletion records (including nested relatedEntities)
|
|
682
826
|
const hasDeletions = this.hasAnyDeletions(records);
|
|
683
827
|
// Write back to file (handles both single records and arrays)
|
|
@@ -703,14 +847,19 @@ export class PushService {
|
|
|
703
847
|
// Drop the cached snapshot — file on disk no longer matches it,
|
|
704
848
|
// and any later reader within this push must see fresh contents.
|
|
705
849
|
this.syncMetadataEngine.invalidateCachedFile(filePath);
|
|
850
|
+
// Non-atomic: this file's records are already committed, so a later failure must
|
|
851
|
+
// not restore the old file and lose their primary keys and sync blocks.
|
|
852
|
+
if (this.writeMode === 'isolated') {
|
|
853
|
+
fileBackupManager.releaseBackup(filePath);
|
|
854
|
+
}
|
|
706
855
|
}
|
|
707
856
|
}
|
|
708
|
-
//
|
|
857
|
+
// Remember the checksum; it is stored only after the push commits (persistIncrementalState).
|
|
709
858
|
// Uses resolved content (after @include) so included-file changes are tracked.
|
|
710
859
|
if (this.stateManager && syncRootDir) {
|
|
711
860
|
const relativePath = path.relative(syncRootDir, filePath);
|
|
712
861
|
const checksum = cachedChecksum ?? this.syncEngine.calculateChecksum(fileData);
|
|
713
|
-
this.
|
|
862
|
+
this.pendingChecksums.set(relativePath, checksum);
|
|
714
863
|
}
|
|
715
864
|
}
|
|
716
865
|
catch (fileError) {
|
|
@@ -718,15 +867,149 @@ export class PushService {
|
|
|
718
867
|
throw fileError;
|
|
719
868
|
}
|
|
720
869
|
}
|
|
721
|
-
//
|
|
870
|
+
// The push timestamp for this directory is stored only after the push commits.
|
|
722
871
|
if (this.stateManager && syncRootDir) {
|
|
723
|
-
|
|
724
|
-
this.stateManager.setLastPushTimestamp(relativeEntityDir, new Date().toISOString());
|
|
725
|
-
await this.stateManager.pruneStaleChecksums(syncRootDir);
|
|
726
|
-
await this.stateManager.save();
|
|
872
|
+
this.pushedEntityDirs.push(path.relative(syncRootDir, entityDir));
|
|
727
873
|
}
|
|
728
874
|
return { created, updated, unchanged, deleted, skipped, deferred, errors };
|
|
729
875
|
}
|
|
876
|
+
/**
|
|
877
|
+
* Run one file's JSON-root graphs, level by level. Atomic mode: one graph at a time on the host.
|
|
878
|
+
* Isolated mode: `graphBatchSize` graphs at once, each on its own independent instance.
|
|
879
|
+
* Fail-fast: the first failed record (thrown or `status: 'error'`) stops the file.
|
|
880
|
+
*/
|
|
881
|
+
async runFileGraphs(run) {
|
|
882
|
+
const pendingWrites = new Map();
|
|
883
|
+
const pool = this.createGraphPool(pendingWrites, run.callbacks);
|
|
884
|
+
pool.noteLevels(run.levels);
|
|
885
|
+
let runError;
|
|
886
|
+
try {
|
|
887
|
+
for (let levelIndex = 0; levelIndex < run.levels.length; levelIndex++) {
|
|
888
|
+
await this.runGraphLevel(run, pool, pendingWrites, levelIndex);
|
|
889
|
+
}
|
|
890
|
+
}
|
|
891
|
+
catch (e) {
|
|
892
|
+
pool.markFailed();
|
|
893
|
+
runError = e;
|
|
894
|
+
}
|
|
895
|
+
const settleError = await pool.releaseAll();
|
|
896
|
+
if (runError)
|
|
897
|
+
throw runError;
|
|
898
|
+
if (settleError)
|
|
899
|
+
throw settleError;
|
|
900
|
+
}
|
|
901
|
+
createGraphPool(pendingWrites, callbacks) {
|
|
902
|
+
return new GraphProviderPool(this.hostProvider(), {
|
|
903
|
+
mode: this.writeMode === 'isolated' ? 'independent' : 'host',
|
|
904
|
+
log: (msg) => callbacks?.onLog?.(msg),
|
|
905
|
+
onGraphSettled: (graphId, outcome) => this.recordGraphOutcome(pendingWrites, graphId, outcome),
|
|
906
|
+
});
|
|
907
|
+
}
|
|
908
|
+
/** Non-atomic mode: a graph whose instance committed leaves its writes in the database. */
|
|
909
|
+
recordGraphOutcome(pendingWrites, graphId, outcome) {
|
|
910
|
+
const writes = pendingWrites.get(graphId);
|
|
911
|
+
pendingWrites.delete(graphId);
|
|
912
|
+
if (writes && outcome === 'committed') {
|
|
913
|
+
this.committedWrites.push(...writes);
|
|
914
|
+
}
|
|
915
|
+
}
|
|
916
|
+
async runGraphLevel(run, pool, pendingWrites, levelIndex) {
|
|
917
|
+
const byGraph = groupRecordsByGraphId(run.levels[levelIndex]);
|
|
918
|
+
const graphIds = Array.from(byGraph.keys());
|
|
919
|
+
const batchSize = this.graphBatchSize;
|
|
920
|
+
if (run.options.verbose && graphIds.length > 1) {
|
|
921
|
+
run.callbacks?.onLog?.(` Level ${levelIndex}: ${run.levels[levelIndex].length} records in ${graphIds.length} graphs (batch ${batchSize}, ${this.writeMode})`);
|
|
922
|
+
}
|
|
923
|
+
for (let i = 0; i < graphIds.length; i += batchSize) {
|
|
924
|
+
const batchIds = graphIds.slice(i, i + batchSize);
|
|
925
|
+
const outcomes = await Promise.all(batchIds.map((graphId) => this.runGraph(run, pool, graphId, byGraph.get(graphId) ?? [])));
|
|
926
|
+
this.applyGraphOutcomes(run, pool, pendingWrites, outcomes.flat());
|
|
927
|
+
const drainError = await pool.drainBatch(batchIds, levelIndex);
|
|
928
|
+
if (drainError)
|
|
929
|
+
throw drainError;
|
|
930
|
+
}
|
|
931
|
+
}
|
|
932
|
+
/** One graph's records, in order, on the graph's provider. Stops at the first thrown error. */
|
|
933
|
+
async runGraph(run, pool, graphId, records) {
|
|
934
|
+
const provider = await pool.obtain(graphId);
|
|
935
|
+
const outcomes = [];
|
|
936
|
+
for (const record of records) {
|
|
937
|
+
try {
|
|
938
|
+
const result = await this.processFlattenedRecord(record, run.entityDir, run.options, run.batchContext, run.callbacks, run.entityConfig, true, provider);
|
|
939
|
+
// Read the depth NOW: a save that settled its own scope is already committed in isolated
|
|
940
|
+
// mode, and that is true whatever the rest of the graph goes on to do.
|
|
941
|
+
outcomes.push({ success: true, result, record, graphId, settled: provider.TransactionDepth === 0 });
|
|
942
|
+
}
|
|
943
|
+
catch (error) {
|
|
944
|
+
pool.markFailed();
|
|
945
|
+
outcomes.push({ success: false, error, record, graphId });
|
|
946
|
+
break;
|
|
947
|
+
}
|
|
948
|
+
}
|
|
949
|
+
return outcomes;
|
|
950
|
+
}
|
|
951
|
+
/**
|
|
952
|
+
* Apply a batch's results. Successful writes are tracked first, so a failure still reports
|
|
953
|
+
* what isolated graphs committed. Then the first thrown error, or any counted record
|
|
954
|
+
* error outside a dry run, stops the push.
|
|
955
|
+
*/
|
|
956
|
+
applyGraphOutcomes(run, pool, pendingWrites, outcomes) {
|
|
957
|
+
let firstFailure;
|
|
958
|
+
for (const outcome of outcomes) {
|
|
959
|
+
if (outcome.success === false) {
|
|
960
|
+
firstFailure ??= outcome;
|
|
961
|
+
continue;
|
|
962
|
+
}
|
|
963
|
+
run.applyResult(outcome.result);
|
|
964
|
+
if (outcome.result.status === 'error') {
|
|
965
|
+
// A non-throwing record error must not commit leftover graph depth.
|
|
966
|
+
pool.markFailed();
|
|
967
|
+
}
|
|
968
|
+
this.trackPendingWrite(pendingWrites, run.filePath, outcome);
|
|
969
|
+
}
|
|
970
|
+
if (firstFailure) {
|
|
971
|
+
this.throwRecordFailure(firstFailure, run.callbacks);
|
|
972
|
+
}
|
|
973
|
+
const errorCount = run.errorCount();
|
|
974
|
+
if (!run.options.dryRun && errorCount > 0) {
|
|
975
|
+
throw new Error(`${errorCount} record${errorCount === 1 ? '' : 's'} in ${path.basename(run.filePath)} could not be pushed ` +
|
|
976
|
+
`(see the errors above). The push stops at the first failed record.`);
|
|
977
|
+
}
|
|
978
|
+
}
|
|
979
|
+
trackPendingWrite(pendingWrites, filePath, outcome) {
|
|
980
|
+
if (this.writeMode !== 'isolated') {
|
|
981
|
+
return; // shared: nothing commits before the push transaction does
|
|
982
|
+
}
|
|
983
|
+
const status = committedStatusOf(outcome.result);
|
|
984
|
+
if (!status) {
|
|
985
|
+
return;
|
|
986
|
+
}
|
|
987
|
+
const write = {
|
|
988
|
+
filePath,
|
|
989
|
+
entityName: outcome.record.entityName,
|
|
990
|
+
recordPath: outcome.record.path,
|
|
991
|
+
status,
|
|
992
|
+
};
|
|
993
|
+
if (outcome.settled) {
|
|
994
|
+
// Committed as it was saved. Reporting it now means a later rollback of the graph's leftover
|
|
995
|
+
// depth cannot make this write disappear from the report while its row is in the database.
|
|
996
|
+
this.committedWrites.push(write);
|
|
997
|
+
return;
|
|
998
|
+
}
|
|
999
|
+
const writes = pendingWrites.get(outcome.graphId) ?? [];
|
|
1000
|
+
writes.push(write);
|
|
1001
|
+
pendingWrites.set(outcome.graphId, writes);
|
|
1002
|
+
}
|
|
1003
|
+
throwRecordFailure(failure, callbacks) {
|
|
1004
|
+
const err = failure.error instanceof Error ? failure.error : new Error(String(failure.error));
|
|
1005
|
+
const rec = failure.record;
|
|
1006
|
+
callbacks?.onLog?.(`\n❌ Processing failed for ${rec.entityName} at ${rec.path}`);
|
|
1007
|
+
callbacks?.onLog?.(` ${err.message}\n`);
|
|
1008
|
+
if (err.stack) {
|
|
1009
|
+
callbacks?.onLog?.(` Stack: ${err.stack}\n`);
|
|
1010
|
+
}
|
|
1011
|
+
throw err;
|
|
1012
|
+
}
|
|
730
1013
|
async processFlattenedRecord(flattenedRecord, entityDir, options, batchContext, callbacks, entityConfig, allowDefer = true, recordProvider) {
|
|
731
1014
|
const metadata = new Metadata(); // global-provider-ok: metadata sync operates on the configured provider only
|
|
732
1015
|
const { record, entityName, parentContext, id: recordId } = flattenedRecord;
|
|
@@ -1227,7 +1510,9 @@ export class PushService {
|
|
|
1227
1510
|
message: errorMessage,
|
|
1228
1511
|
});
|
|
1229
1512
|
// Throw error to trigger rollback and stop processing
|
|
1230
|
-
|
|
1513
|
+
const saveFailure = new Error(`Failed to save ${entityName} record at ${flattenedRecord.path}: ${errorMessage}`);
|
|
1514
|
+
this.reportedRecordErrors.add(saveFailure);
|
|
1515
|
+
throw saveFailure;
|
|
1231
1516
|
}
|
|
1232
1517
|
// Return batch context entry as a side effect instead of mutating directly.
|
|
1233
1518
|
// The caller applies this sequentially after Promise.all() resolves, avoiding
|
|
@@ -1523,7 +1808,9 @@ export class PushService {
|
|
|
1523
1808
|
messages.push(' • No deletions');
|
|
1524
1809
|
}
|
|
1525
1810
|
messages.push('');
|
|
1526
|
-
|
|
1811
|
+
for (const line of this.transactionBannerLines()) {
|
|
1812
|
+
messages.push(line);
|
|
1813
|
+
}
|
|
1527
1814
|
messages.push('');
|
|
1528
1815
|
messages.push('═'.repeat(80));
|
|
1529
1816
|
messages.push('');
|
|
@@ -1547,6 +1834,19 @@ export class PushService {
|
|
|
1547
1834
|
* Audit all deletions across all metadata files
|
|
1548
1835
|
* This pre-processes all records to identify deletion dependencies and order
|
|
1549
1836
|
*/
|
|
1837
|
+
/** What the deletion confirmation says about rollback. Must match the write mode. */
|
|
1838
|
+
transactionBannerLines() {
|
|
1839
|
+
const isolated = [...this.directoryModes.entries()].filter(([, mode]) => mode === 'isolated');
|
|
1840
|
+
if (isolated.length === 0) {
|
|
1841
|
+
return ['All creates, updates and deletes run in one database transaction. If anything fails, nothing is saved.'];
|
|
1842
|
+
}
|
|
1843
|
+
const names = isolated.map(([dir]) => path.relative(process.cwd(), dir) || dir).join(', ');
|
|
1844
|
+
return [
|
|
1845
|
+
'Deletes and deferred records run in one database transaction and are rolled back on error.',
|
|
1846
|
+
`These directories use isolated transactions, so each create and update in them is committed as soon as it is`,
|
|
1847
|
+
`saved and is NOT rolled back: ${names}.`,
|
|
1848
|
+
];
|
|
1849
|
+
}
|
|
1550
1850
|
async auditAllDeletions(entityDirs, options, callbacks) {
|
|
1551
1851
|
// OPTIMIZATION: Quick scan to check if ANY deletions exist before doing expensive loading
|
|
1552
1852
|
let hasAnyDeletions = false;
|
|
@@ -1838,11 +2138,23 @@ export class PushService {
|
|
|
1838
2138
|
}
|
|
1839
2139
|
}
|
|
1840
2140
|
catch (error) {
|
|
1841
|
-
const err = error;
|
|
2141
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
1842
2142
|
callbacks?.onError?.(` ✗ Failed to process deferred record: ${entityName} (${recordId})`);
|
|
1843
2143
|
callbacks?.onError?.(` Error: ${err.message}`);
|
|
1844
2144
|
callbacks?.onError?.(` Tip: Ensure all referenced records exist or remove the ?allowDefer flag`);
|
|
2145
|
+
if (!this.reportedRecordErrors.has(err)) {
|
|
2146
|
+
callbacks?.onRecordError?.({
|
|
2147
|
+
entityName,
|
|
2148
|
+
path: flattenedRecord.path,
|
|
2149
|
+
primaryKey: recordId,
|
|
2150
|
+
message: `Deferred record could not be resolved: ${err.message}`,
|
|
2151
|
+
});
|
|
2152
|
+
}
|
|
1845
2153
|
errors++;
|
|
2154
|
+
if (!options.dryRun) {
|
|
2155
|
+
// Same as a thrown Phase 1 error: stop, and let push() roll the transaction back.
|
|
2156
|
+
throw new Error(`Failed to process deferred record ${entityName} (${recordId}): ${err.message}`, { cause: err });
|
|
2157
|
+
}
|
|
1846
2158
|
}
|
|
1847
2159
|
}
|
|
1848
2160
|
// Summary
|