@memberjunction/metadata-sync 6.1.2 → 6.2.0-edge.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/README.md +88 -25
  2. package/dist/config.d.ts +42 -2
  3. package/dist/config.d.ts.map +1 -1
  4. package/dist/config.js.map +1 -1
  5. package/dist/index.d.ts +6 -0
  6. package/dist/index.d.ts.map +1 -1
  7. package/dist/index.js +3 -0
  8. package/dist/index.js.map +1 -1
  9. package/dist/lib/EntityPropertyExtractor.d.ts +16 -0
  10. package/dist/lib/EntityPropertyExtractor.d.ts.map +1 -1
  11. package/dist/lib/EntityPropertyExtractor.js +34 -0
  12. package/dist/lib/EntityPropertyExtractor.js.map +1 -1
  13. package/dist/lib/FieldExternalizer.d.ts +1 -2
  14. package/dist/lib/FieldExternalizer.d.ts.map +1 -1
  15. package/dist/lib/FieldExternalizer.js.map +1 -1
  16. package/dist/lib/RecordProcessor.d.ts +7 -0
  17. package/dist/lib/RecordProcessor.d.ts.map +1 -1
  18. package/dist/lib/RecordProcessor.js +27 -1
  19. package/dist/lib/RecordProcessor.js.map +1 -1
  20. package/dist/lib/existing-record-files.d.ts +12 -0
  21. package/dist/lib/existing-record-files.d.ts.map +1 -0
  22. package/dist/lib/existing-record-files.js +85 -0
  23. package/dist/lib/existing-record-files.js.map +1 -0
  24. package/dist/lib/file-backup-manager.d.ts +10 -0
  25. package/dist/lib/file-backup-manager.d.ts.map +1 -1
  26. package/dist/lib/file-backup-manager.js +17 -0
  27. package/dist/lib/file-backup-manager.js.map +1 -1
  28. package/dist/lib/file-write-batch.d.ts +0 -4
  29. package/dist/lib/file-write-batch.d.ts.map +1 -1
  30. package/dist/lib/file-write-batch.js +10 -8
  31. package/dist/lib/file-write-batch.js.map +1 -1
  32. package/dist/lib/graph-provider-pool.d.ts +62 -33
  33. package/dist/lib/graph-provider-pool.d.ts.map +1 -1
  34. package/dist/lib/graph-provider-pool.js +131 -79
  35. package/dist/lib/graph-provider-pool.js.map +1 -1
  36. package/dist/lib/json-subproperty-externalization.d.ts +77 -0
  37. package/dist/lib/json-subproperty-externalization.d.ts.map +1 -0
  38. package/dist/lib/json-subproperty-externalization.js +139 -0
  39. package/dist/lib/json-subproperty-externalization.js.map +1 -0
  40. package/dist/lib/push-outcome.d.ts +66 -0
  41. package/dist/lib/push-outcome.d.ts.map +1 -0
  42. package/dist/lib/push-outcome.js +81 -0
  43. package/dist/lib/push-outcome.js.map +1 -0
  44. package/dist/lib/push-write-mode.d.ts +47 -0
  45. package/dist/lib/push-write-mode.d.ts.map +1 -0
  46. package/dist/lib/push-write-mode.js +53 -0
  47. package/dist/lib/push-write-mode.js.map +1 -0
  48. package/dist/lib/record-primary-key.d.ts +35 -0
  49. package/dist/lib/record-primary-key.d.ts.map +1 -0
  50. package/dist/lib/record-primary-key.js +44 -0
  51. package/dist/lib/record-primary-key.js.map +1 -0
  52. package/dist/lib/transaction-manager.d.ts +11 -1
  53. package/dist/lib/transaction-manager.d.ts.map +1 -1
  54. package/dist/lib/transaction-manager.js +20 -4
  55. package/dist/lib/transaction-manager.js.map +1 -1
  56. package/dist/plugins/index.d.ts +1 -0
  57. package/dist/plugins/index.d.ts.map +1 -1
  58. package/dist/plugins/index.js +43 -17
  59. package/dist/plugins/index.js.map +1 -1
  60. package/dist/services/PullService.d.ts +1 -3
  61. package/dist/services/PullService.d.ts.map +1 -1
  62. package/dist/services/PullService.js +28 -68
  63. package/dist/services/PullService.js.map +1 -1
  64. package/dist/services/PushService.d.ts +109 -0
  65. package/dist/services/PushService.d.ts.map +1 -1
  66. package/dist/services/PushService.js +631 -319
  67. package/dist/services/PushService.js.map +1 -1
  68. package/package.json +14 -14
@@ -15,18 +15,40 @@ import { SQLLogger } from '../lib/sql-logger.js';
15
15
  import { TransactionManager } from '../lib/transaction-manager.js';
16
16
  import { JsonWriteHelper } from '../lib/json-write-helper.js';
17
17
  import { RecordDependencyAnalyzer, groupRecordsByGraphId } from '../lib/record-dependency-analyzer.js';
18
- import { GraphProviderPool } from '../lib/graph-provider-pool.js';
18
+ import { GraphProviderPool, probeIndependentInstances } from '../lib/graph-provider-pool.js';
19
+ import { resolveDirectoryMode, graphBatchSizeFor, isolatedModeWarning, unusedBatchSizeWarning, } from '../lib/push-write-mode.js';
20
+ import { PushAbortedError, describeCommitFailure, describeRollbackOutcome } from '../lib/push-outcome.js';
19
21
  import { JsonPreprocessor } from '../lib/json-preprocessor.js';
20
22
  import { findEntityDirectories } from '../lib/provider-utils.js';
21
23
  import { DeletionAuditor } from '../lib/deletion-auditor.js';
22
24
  import { describeMissingEntitySubclass } from '../lib/entity-subclass-guard.js';
23
25
  import { DeletionReportGenerator } from '../lib/deletion-report-generator.js';
24
26
  import { resolveCollectionRelationship } from '../lib/collection-resolver.js';
25
- // Parallelism is across JSON-root graphs (independent Actions), not flattened rows.
26
- // Nested relatedEntities share the root's provider so parent+child stay on one TX.
27
- // Default 10 — never 1. 1 was a wrong workaround for mixed-provider hangs;
28
- // callers can pass --parallel-batch-size 1 for debugging.
29
- const PARALLEL_BATCH_SIZE = 10;
27
+ function emptyPushTotals() {
28
+ return { created: 0, updated: 0, unchanged: 0, deleted: 0, skipped: 0, deferred: 0, errors: 0 };
29
+ }
30
+ function addPushTotals(totals, result) {
31
+ totals.created += result.created;
32
+ totals.updated += result.updated;
33
+ totals.unchanged += result.unchanged;
34
+ totals.deleted += result.deleted;
35
+ totals.skipped += result.skipped;
36
+ totals.deferred += result.deferred;
37
+ totals.errors += result.errors;
38
+ }
39
+ /** The status a committed write is reported with, or undefined when the result wrote nothing. */
40
+ function committedStatusOf(result) {
41
+ if (result.isDuplicate || result.isDeletedRecord) {
42
+ return undefined;
43
+ }
44
+ if (result.status === 'updated') {
45
+ return 'updated';
46
+ }
47
+ if (result.status === 'created' || result.status === 'deferred') {
48
+ return 'created';
49
+ }
50
+ return undefined;
51
+ }
30
52
  export class PushService {
31
53
  constructor(syncEngine, contextUser, stateManager) {
32
54
  this.warnings = [];
@@ -35,6 +57,20 @@ export class PushService {
35
57
  this.deferredFileWrites = new Map();
36
58
  this.deferredRecords = [];
37
59
  this.confirmedCollections = new Set();
60
+ /** Write mode per entity directory, resolved once at the start of the push. */
61
+ this.directoryModes = new Map();
62
+ /** The directory being processed right now, and how it writes. */
63
+ this.writeMode = 'shared';
64
+ this.graphBatchSize = 1;
65
+ /** Counts so far, so a failed push can still report what it did. */
66
+ this.runningTotals = emptyPushTotals();
67
+ /** Non-atomic mode: creates and updates that were committed outside the push transaction. */
68
+ this.committedWrites = [];
69
+ /** Incremental state, applied only after the push commits. Keyed by path relative to the sync root. */
70
+ this.pendingChecksums = new Map();
71
+ this.pushedEntityDirs = [];
72
+ /** Errors whose record was already sent to onRecordError, so a caller does not report it twice. */
73
+ this.reportedRecordErrors = new WeakSet();
38
74
  this.syncEngine = syncEngine;
39
75
  this.contextUser = contextUser;
40
76
  this.stateManager = stateManager;
@@ -133,9 +169,15 @@ export class PushService {
133
169
  if (options.include && options.exclude) {
134
170
  throw new Error('Cannot specify both --include and --exclude options. Please use one or the other.');
135
171
  }
136
- // Reset deferred tracking for this push operation
172
+ // Reset per-push tracking
137
173
  this.deferredFileWrites.clear();
138
174
  this.deferredRecords = [];
175
+ this.committedWrites = [];
176
+ this.pendingChecksums.clear();
177
+ this.pushedEntityDirs = [];
178
+ this.directoryModes.clear();
179
+ this.runningTotals = emptyPushTotals();
180
+ this.sqlLogFilePath = undefined;
139
181
  const fileBackupManager = new FileBackupManager();
140
182
  // Load sync config for SQL logging settings and autoCreateMissingRecords flag
141
183
  // If dir option is specified, load from that directory, otherwise use original CWD
@@ -156,54 +198,15 @@ export class PushService {
156
198
  callbacks?.onLog?.(`SQL logging config: ${JSON.stringify(this.syncConfig?.sqlLogging)}`);
157
199
  }
158
200
  const sqlLogger = new SQLLogger(this.syncConfig);
159
- const transactionManager = new TransactionManager(sqlLogger);
201
+ // The push transaction lives on the same provider every atomic save runs on.
202
+ const transactionManager = new TransactionManager(sqlLogger, this.hostProvider());
160
203
  if (options.verbose) {
161
204
  callbacks?.onLog?.(`SQLLogger enabled status: ${sqlLogger.enabled}`);
162
205
  }
163
206
  // Setup SQL logging session with the provider if enabled
164
207
  let sqlLoggingSession = null;
165
208
  try {
166
- // Initialize SQL logger if enabled and not dry-run
167
- if (sqlLogger.enabled && !options.dryRun) {
168
- const provider = Metadata.Provider; // global-provider-ok: metadata sync operates on the configured provider only
169
- if (options.verbose) {
170
- callbacks?.onLog?.(`SQL logging enabled: ${sqlLogger.enabled}`);
171
- callbacks?.onLog?.(`Provider type: ${provider?.constructor?.name || 'Unknown'}`);
172
- callbacks?.onLog?.(`Has CreateSqlLogger: ${typeof provider?.CreateSqlLogger === 'function'}`);
173
- }
174
- if (provider && typeof provider.CreateSqlLogger === 'function') {
175
- // Generate filename with timestamp
176
- const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
177
- const filename = this.syncConfig?.sqlLogging?.formatAsMigration
178
- ? `MetadataSync_Push_${timestamp}.sql`
179
- : `push_${timestamp}.sql`;
180
- // Use .sql-log-push directory in the config directory (where sync was initiated)
181
- const outputDir = path.join(configDir, this.syncConfig?.sqlLogging?.outputDirectory || './sql-log-push');
182
- const filepath = path.join(outputDir, filename);
183
- // Ensure the directory exists
184
- await fs.ensureDir(path.dirname(filepath));
185
- // Create the SQL logging session
186
- sqlLoggingSession = await provider.CreateSqlLogger(filepath, {
187
- formatAsMigration: this.syncConfig?.sqlLogging?.formatAsMigration || false,
188
- description: 'MetadataSync push operation',
189
- statementTypes: "mutations",
190
- prettyPrint: true,
191
- // batchSeparator is intentionally omitted — CreateSqlLogger injects the platform-appropriate
192
- // separator automatically (GO for SQL Server, nothing for PostgreSQL) via PlatformBatchSeparator.
193
- filterPatterns: this.syncConfig?.sqlLogging?.filterPatterns,
194
- filterType: this.syncConfig?.sqlLogging?.filterType,
195
- verboseOutput: this.syncConfig?.sqlLogging?.verboseOutput || false,
196
- });
197
- if (options.verbose) {
198
- callbacks?.onLog?.(`📝 SQL logging enabled: ${filepath}`);
199
- }
200
- }
201
- else {
202
- if (options.verbose) {
203
- callbacks?.onWarn?.('SQL logging requested but provider does not support it');
204
- }
205
- }
206
- }
209
+ sqlLoggingSession = await this.startSqlLogging(sqlLogger, options, configDir, callbacks);
207
210
  // Find entity directories to process
208
211
  // Note: If options.dir is specified, configDir already points to that directory
209
212
  // So we don't need to pass it as specificDir
@@ -211,32 +214,8 @@ export class PushService {
211
214
  if (entityDirs.length === 0) {
212
215
  throw new Error('No entity directories found');
213
216
  }
214
- // Preload entities and cache files.
215
- // We pass the SyncEngine's provider explicitly rather than reaching for
216
- // Metadata.Provider — `mj sync` is single-process so they resolve to the
217
- // same instance today, but routing through SyncEngine keeps the wiring
218
- // self-consistent and makes future provider plumbing trivial.
219
- // Preload is internal plumbing — emit its progress only in verbose mode so a
220
- // normal run jumps straight from validation to per-directory results.
221
- if (options.verbose) {
222
- callbacks?.onLog?.('⚡ Preloading metadata and caching files...');
223
- }
224
- this.syncMetadataEngine.setEntityDirs(entityDirs);
225
- await this.syncMetadataEngine.Config(true, this.contextUser, this.syncEngine.getProvider());
226
- for (const warning of this.syncMetadataEngine.drainWarnings()) {
227
- callbacks?.onWarn?.(` ⚠️ ${warning}`);
228
- }
229
- if (options.verbose) {
230
- const delegations = this.syncMetadataEngine.getDelegationSummary();
231
- if (delegations.length > 0) {
232
- const donorCount = new Set(delegations.map(d => d.engineClassName)).size;
233
- callbacks?.onLog?.(` ↪ Reused in-memory caches for ${delegations.length} ${delegations.length === 1 ? 'entity' : 'entities'} already loaded by ${donorCount} ${donorCount === 1 ? 'engine' : 'engines'}`);
234
- for (const d of delegations.sort((a, b) => a.entityName.localeCompare(b.entityName))) {
235
- callbacks?.onLog?.(` • ${d.entityName} ← ${d.engineClassName}`);
236
- }
237
- }
238
- callbacks?.onLog?.('✓ Preload completed successfully\n');
239
- }
217
+ await this.applyWritePlan(entityDirs, options, callbacks);
218
+ await this.preloadMetadata(entityDirs, options, callbacks);
240
219
  if (options.verbose) {
241
220
  callbacks?.onLog?.(`Found ${entityDirs.length} entity ${entityDirs.length === 1 ? 'directory' : 'directories'} to process`);
242
221
  }
@@ -247,14 +226,6 @@ export class PushService {
247
226
  callbacks?.onLog?.('📁 File backup manager initialized');
248
227
  }
249
228
  }
250
- // Process each entity directory
251
- let totalCreated = 0;
252
- let totalUpdated = 0;
253
- let totalUnchanged = 0;
254
- let totalDeleted = 0;
255
- let totalSkipped = 0;
256
- let totalDeferred = 0;
257
- let totalErrors = 0;
258
229
  // PHASE 0: Audit all deletions across all entities (if any exist)
259
230
  let deletionAudit = null;
260
231
  try {
@@ -268,24 +239,7 @@ export class PushService {
268
239
  if (!options.dryRun && deletionAudit) {
269
240
  const shouldProceed = await this.promptForConfirmation(deletionAudit, callbacks);
270
241
  if (!shouldProceed) {
271
- callbacks?.onLog?.('\n❌ Push operation cancelled by user.\n');
272
- // Clean up SQL logging session and file if it was created
273
- if (sqlLoggingSession) {
274
- const sqlLogPath = sqlLoggingSession.filePath;
275
- try {
276
- await sqlLoggingSession.dispose();
277
- // Delete the empty SQL log file since no operations occurred
278
- if (await fs.pathExists(sqlLogPath)) {
279
- await fs.remove(sqlLogPath);
280
- if (options.verbose) {
281
- callbacks?.onLog?.(`🗑️ Removed empty SQL log file: ${sqlLogPath}`);
282
- }
283
- }
284
- }
285
- catch (cleanupError) {
286
- callbacks?.onWarn?.(`Failed to clean up SQL logging session: ${cleanupError}`);
287
- }
288
- }
242
+ await this.cancelPush(sqlLoggingSession, options, callbacks);
289
243
  return {
290
244
  created: 0,
291
245
  updated: 0,
@@ -299,128 +253,14 @@ export class PushService {
299
253
  };
300
254
  }
301
255
  }
302
- // Host TX wraps Phase 2 deletions and Phase 2.5 deferred records.
303
- // Phase 1 graph writes go to independent instances (or, if those are
304
- // unavailable, ALL graphs share this host TX — never a mix).
256
+ // One host transaction wraps the whole push. In atomic mode (the default) Phase 1 saves run
257
+ // on the host too, one graph at a time. A directory using isolated transactions commits on
258
+ // independent instances as it goes (see lib/push-write-mode.ts).
305
259
  if (!options.dryRun) {
306
260
  await transactionManager.beginTransaction();
307
261
  }
308
- try {
309
- // PHASE 1: Process creates/updates for all entities
310
- if (options.verbose) {
311
- callbacks?.onLog?.('📝 Processing creates and updates...\n');
312
- }
313
- for (const [dirIdx, entityDir] of entityDirs.entries()) {
314
- // "X of N" position prefix — only when there's more than one directory, so a
315
- // single-directory push stays uncluttered.
316
- const progressPrefix = entityDirs.length > 1 ? `[${dirIdx + 1}/${entityDirs.length}] ` : '';
317
- const entityConfig = await loadEntityConfig(entityDir);
318
- if (!entityConfig) {
319
- const warning = `Skipping ${entityDir} - no valid entity configuration`;
320
- this.warnings.push(warning);
321
- callbacks?.onWarn?.(warning);
322
- totalSkipped++; // Count skipped directories
323
- continue;
324
- }
325
- // Show folder with spinner at start. The folder header is redundant in a
326
- // normal run (the per-directory result line below names the directory), so
327
- // it's verbose-only; the live spinner still shows "[X/N] Processing <dir>…".
328
- const dirName = path.relative(process.cwd(), entityDir) || '.';
329
- if (options.verbose) {
330
- callbacks?.onLog?.(`\n📁 ${dirName}:`);
331
- }
332
- // Use onProgress for animated spinner if available
333
- if (callbacks?.onProgress) {
334
- callbacks.onProgress(`${progressPrefix}Processing ${dirName}...`);
335
- }
336
- else {
337
- callbacks?.onLog?.(` ⏳ Processing...`);
338
- }
339
- if (options.verbose && callbacks?.onLog) {
340
- callbacks.onLog(`Processing ${entityConfig.entity} in ${entityDir}`);
341
- }
342
- const result = await this.processEntityDirectory(entityDir, entityConfig, options, fileBackupManager, callbacks, configDir);
343
- // Per-directory result: one compact line (always), naming the directory and
344
- // its changes — or "no changes" for a clean dir. The detailed per-status
345
- // breakdown is verbose-only since the final summary box already aggregates it.
346
- const dirTotal = result.created + result.updated + result.unchanged + result.deleted + result.skipped;
347
- const { text: dirSummary, changed: dirChanged } = this.formatDirectorySummary(progressPrefix, dirName, result, dirTotal);
348
- if (callbacks?.onProgress && callbacks?.onSuccess) {
349
- callbacks.onSuccess(dirSummary, dirChanged);
350
- }
351
- else {
352
- callbacks?.onLog?.(` ${dirSummary}`);
353
- }
354
- if (options.verbose && (dirTotal > 0 || result.errors > 0)) {
355
- callbacks?.onLog?.(` Total processed: ${dirTotal} records`);
356
- if (result.created > 0) {
357
- callbacks?.onLog?.(` ✓ Created: ${result.created}`);
358
- }
359
- if (result.updated > 0) {
360
- callbacks?.onLog?.(` ✓ Updated: ${result.updated}`);
361
- }
362
- if (result.deleted > 0) {
363
- callbacks?.onLog?.(` ✓ Deleted: ${result.deleted}`);
364
- }
365
- if (result.deferred > 0) {
366
- callbacks?.onLog?.(` ⏳ Deferred: ${result.deferred}`);
367
- }
368
- if (result.unchanged > 0) {
369
- callbacks?.onLog?.(` - Unchanged: ${result.unchanged}`);
370
- }
371
- if (result.skipped > 0) {
372
- callbacks?.onLog?.(` - Skipped: ${result.skipped}`);
373
- }
374
- if (result.errors > 0) {
375
- callbacks?.onLog?.(` ✗ Errors: ${result.errors}`);
376
- }
377
- }
378
- totalCreated += result.created;
379
- totalUpdated += result.updated;
380
- totalUnchanged += result.unchanged;
381
- totalDeleted += result.deleted;
382
- totalSkipped += result.skipped;
383
- totalDeferred += result.deferred;
384
- totalErrors += result.errors;
385
- }
386
- // PHASE 2: Process deletions in reverse dependency order (if any exist)
387
- if (deletionAudit && totalErrors === 0) {
388
- const deletionResult = await this.processDeletionsFromAudit(deletionAudit, options, callbacks);
389
- totalDeleted += deletionResult.deleted;
390
- totalErrors += deletionResult.errors;
391
- }
392
- // PHASE 2.5: Process deferred records (for circular dependencies)
393
- if (this.deferredRecords.length > 0 && totalErrors === 0) {
394
- const deferredResult = await this.processDeferredRecords(options, callbacks);
395
- totalCreated += deferredResult.created;
396
- totalUpdated += deferredResult.updated;
397
- totalErrors += deferredResult.errors;
398
- }
399
- // Commit transaction if successful
400
- if (!options.dryRun && totalErrors === 0) {
401
- await transactionManager.commitTransaction();
402
- }
403
- // PHASE 3: Write deferred files with updated deletion timestamps
404
- if (!options.dryRun && totalErrors === 0 && this.deferredFileWrites.size > 0) {
405
- await this.writeDeferredFiles(options, callbacks);
406
- }
407
- }
408
- catch (error) {
409
- // Rollback transaction on error.
410
- if (!options.dryRun) {
411
- callbacks?.onLog?.('\n⚠️ Rolling back database transaction due to error...');
412
- const rolledBack = await transactionManager.rollbackTransaction();
413
- if (rolledBack) {
414
- callbacks?.onLog?.('✓ Database transaction rolled back successfully\n');
415
- }
416
- else {
417
- callbacks?.onLog?.('❌ Database transaction rollback failed\n');
418
- }
419
- }
420
- throw error;
421
- }
422
- // Commit file backups if successful and not in dry-run mode
423
- if (!options.dryRun && totalErrors === 0) {
262
+ const totals = await this.runPushInTransaction({ entityDirs, deletionAudit, options, fileBackupManager, callbacks, configDir }, transactionManager);
263
+ if (!options.dryRun) {
424
264
  await fileBackupManager.cleanup();
425
265
  if (options.verbose) {
426
266
  callbacks?.onLog?.('✅ File backups committed');
@@ -443,13 +283,7 @@ export class PushService {
443
283
  }
444
284
  }
445
285
  return {
446
- created: totalCreated,
447
- updated: totalUpdated,
448
- unchanged: totalUnchanged,
449
- deleted: totalDeleted,
450
- skipped: totalSkipped,
451
- deferred: totalDeferred,
452
- errors: totalErrors,
286
+ ...totals,
453
287
  warnings: this.warnings,
454
288
  sqlLogPath,
455
289
  changeLog: this.changeDetails
@@ -478,6 +312,381 @@ export class PushService {
478
312
  throw error;
479
313
  }
480
314
  }
315
+ /** Open the SQL logging session when the config asks for one and this is not a dry run. */
316
+ async startSqlLogging(sqlLogger, options, configDir, callbacks) {
317
+ let sqlLoggingSession = null;
318
+ if (sqlLogger.enabled && !options.dryRun) {
319
+ const provider = Metadata.Provider; // global-provider-ok: metadata sync operates on the configured provider only
320
+ if (options.verbose) {
321
+ callbacks?.onLog?.(`SQL logging enabled: ${sqlLogger.enabled}`);
322
+ callbacks?.onLog?.(`Provider type: ${provider?.constructor?.name || 'Unknown'}`);
323
+ callbacks?.onLog?.(`Has CreateSqlLogger: ${typeof provider?.CreateSqlLogger === 'function'}`);
324
+ }
325
+ if (provider && typeof provider.CreateSqlLogger === 'function') {
326
+ // Generate filename with timestamp
327
+ const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
328
+ const filename = this.syncConfig?.sqlLogging?.formatAsMigration
329
+ ? `MetadataSync_Push_${timestamp}.sql`
330
+ : `push_${timestamp}.sql`;
331
+ // Use .sql-log-push directory in the config directory (where sync was initiated)
332
+ const outputDir = path.join(configDir, this.syncConfig?.sqlLogging?.outputDirectory || './sql-log-push');
333
+ const filepath = path.join(outputDir, filename);
334
+ // Ensure the directory exists
335
+ await fs.ensureDir(path.dirname(filepath));
336
+ // Create the SQL logging session
337
+ this.sqlLogFilePath = filepath;
338
+ sqlLoggingSession = await provider.CreateSqlLogger(filepath, {
339
+ formatAsMigration: this.syncConfig?.sqlLogging?.formatAsMigration || false,
340
+ description: 'MetadataSync push operation',
341
+ statementTypes: "mutations",
342
+ prettyPrint: true,
343
+ // batchSeparator is intentionally omitted — CreateSqlLogger injects the platform-appropriate
344
+ // separator automatically (GO for SQL Server, nothing for PostgreSQL) via PlatformBatchSeparator.
345
+ filterPatterns: this.syncConfig?.sqlLogging?.filterPatterns,
346
+ filterType: this.syncConfig?.sqlLogging?.filterType,
347
+ verboseOutput: this.syncConfig?.sqlLogging?.verboseOutput || false,
348
+ });
349
+ if (options.verbose) {
350
+ callbacks?.onLog?.(`📝 SQL logging enabled: ${filepath}`);
351
+ }
352
+ }
353
+ else {
354
+ if (options.verbose) {
355
+ callbacks?.onWarn?.('SQL logging requested but provider does not support it');
356
+ }
357
+ }
358
+ }
359
+ return sqlLoggingSession;
360
+ }
361
+ /**
362
+ * Preload entity metadata and cache the files on disk.
363
+ *
364
+ * The SyncEngine's provider is passed explicitly rather than reaching for Metadata.Provider —
365
+ * `mj sync` is single-process so they resolve to the same instance today, but routing through
366
+ * SyncEngine keeps the wiring self-consistent.
367
+ */
368
+ async preloadMetadata(entityDirs, options, callbacks) {
369
+ // We pass the SyncEngine's provider explicitly rather than reaching for
370
+ // Metadata.Provider — `mj sync` is single-process so they resolve to the
371
+ // same instance today, but routing through SyncEngine keeps the wiring
372
+ // self-consistent and makes future provider plumbing trivial.
373
+ // Preload is internal plumbing — emit its progress only in verbose mode so a
374
+ // normal run jumps straight from validation to per-directory results.
375
+ if (options.verbose) {
376
+ callbacks?.onLog?.('⚡ Preloading metadata and caching files...');
377
+ }
378
+ this.syncMetadataEngine.setEntityDirs(entityDirs);
379
+ await this.syncMetadataEngine.Config(true, this.contextUser, this.syncEngine.getProvider());
380
+ for (const warning of this.syncMetadataEngine.drainWarnings()) {
381
+ callbacks?.onWarn?.(` ⚠️ ${warning}`);
382
+ }
383
+ if (options.verbose) {
384
+ const delegations = this.syncMetadataEngine.getDelegationSummary();
385
+ if (delegations.length > 0) {
386
+ const donorCount = new Set(delegations.map(d => d.engineClassName)).size;
387
+ callbacks?.onLog?.(` ↪ Reused in-memory caches for ${delegations.length} ${delegations.length === 1 ? 'entity' : 'entities'} already loaded by ${donorCount} ${donorCount === 1 ? 'engine' : 'engines'}`);
388
+ for (const d of delegations.sort((a, b) => a.entityName.localeCompare(b.entityName))) {
389
+ callbacks?.onLog?.(` • ${d.entityName} ← ${d.engineClassName}`);
390
+ }
391
+ }
392
+ callbacks?.onLog?.('✓ Preload completed successfully\n');
393
+ }
394
+ }
395
+ /** The user declined the deletion prompt: drop an empty SQL log and say nothing happened. */
396
+ async cancelPush(sqlLoggingSession, options, callbacks) {
397
+ callbacks?.onLog?.('\n❌ Push operation cancelled by user.\n');
398
+ // Clean up SQL logging session and file if it was created
399
+ if (sqlLoggingSession) {
400
+ const sqlLogPath = sqlLoggingSession.filePath;
401
+ try {
402
+ await sqlLoggingSession.dispose();
403
+ // Delete the empty SQL log file since no operations occurred
404
+ if (await fs.pathExists(sqlLogPath)) {
405
+ await fs.remove(sqlLogPath);
406
+ if (options.verbose) {
407
+ callbacks?.onLog?.(`🗑️ Removed empty SQL log file: ${sqlLogPath}`);
408
+ }
409
+ }
410
+ }
411
+ catch (cleanupError) {
412
+ callbacks?.onWarn?.(`Failed to clean up SQL logging session: ${cleanupError}`);
413
+ }
414
+ }
415
+ }
416
+ /**
417
+ * Decide, once and up front, how each entity directory writes: shared by default, isolated where
418
+ * the entity (or the root, or the CLI flag) asks for it. Deciding it here means a push can never
419
+ * discover mid-file that it must change topology, which is the mixed-provider deadlock.
420
+ */
421
+ async applyWritePlan(entityDirs, options, callbacks) {
422
+ const isolated = [];
423
+ for (const entityDir of entityDirs) {
424
+ const entityConfig = await loadEntityConfig(entityDir);
425
+ const resolved = resolveDirectoryMode({
426
+ isolatedFlag: options.isolatedTransactions,
427
+ entityIsolated: entityConfig?.push?.isolatedTransactions,
428
+ rootIsolated: this.syncConfig?.push?.isolatedTransactions,
429
+ });
430
+ this.directoryModes.set(entityDir, resolved.mode);
431
+ if (resolved.mode === 'isolated') {
432
+ isolated.push(path.relative(process.cwd(), entityDir) || entityDir);
433
+ }
434
+ if (options.verbose) {
435
+ callbacks?.onLog?.(` ${path.relative(process.cwd(), entityDir) || entityDir}: ${resolved.mode} (from ${resolved.source})`);
436
+ }
437
+ }
438
+ if (isolated.length > 0) {
439
+ await this.confirmIsolatedTransactions(isolated, options, callbacks);
440
+ }
441
+ else if (options.parallelBatchSize !== undefined && options.parallelBatchSize !== 1) {
442
+ this.addWarning(unusedBatchSizeWarning(options.parallelBatchSize), callbacks);
443
+ }
444
+ }
445
+ /**
446
+ * Isolation needs independent provider instances. Without them every directory runs shared,
447
+ * because the alternative — some graphs on the host, some on their own connection — is the
448
+ * deadlock the graph pool exists to prevent.
449
+ */
450
+ async confirmIsolatedTransactions(isolated, options, callbacks) {
451
+ const reason = await probeIndependentInstances(this.hostProvider());
452
+ if (reason) {
453
+ this.addWarning(`Independent provider instances are not available (${reason}), so every directory runs in the shared ` +
454
+ `push transaction: one transaction, one graph at a time.`, callbacks);
455
+ for (const dir of this.directoryModes.keys()) {
456
+ this.directoryModes.set(dir, 'shared');
457
+ }
458
+ return;
459
+ }
460
+ if (!options.dryRun) {
461
+ this.addWarning(isolatedModeWarning(isolated, graphBatchSizeFor('isolated', options.parallelBatchSize)), callbacks);
462
+ }
463
+ }
464
+ /** Adopt one directory's mode for the work about to run in it. */
465
+ useDirectoryMode(entityDir, options) {
466
+ this.writeMode = this.directoryModes.get(entityDir) ?? 'shared';
467
+ this.graphBatchSize = graphBatchSizeFor(this.writeMode, options.parallelBatchSize);
468
+ }
469
+ addWarning(message, callbacks) {
470
+ this.warnings.push(message);
471
+ callbacks?.onWarn?.(`⚠️ ${message}`);
472
+ }
473
+ /** The process-wide provider that owns the push transaction. */
474
+ hostProvider() {
475
+ return Metadata.Provider; // global-provider-ok: mj sync is single-process; this provider owns the push transaction
476
+ }
477
+ /**
478
+ * Run every phase inside the push transaction, commit it, then do the post-commit work.
479
+ * Never returns or throws with the transaction still open.
480
+ */
481
+ async runPushInTransaction(run, transactionManager) {
482
+ let committed = false;
483
+ try {
484
+ const totals = await this.runPushPhases(run);
485
+ if (!run.options.dryRun) {
486
+ await this.commitPushTransaction(transactionManager);
487
+ committed = true;
488
+ // PHASE 3: files that contained deletions, then the incremental state
489
+ await this.writeDeferredFiles(run.options, run.callbacks);
490
+ await this.persistIncrementalState(run.configDir);
491
+ }
492
+ return totals;
493
+ }
494
+ catch (error) {
495
+ if (committed) {
496
+ throw await this.failAfterCommit(error, run);
497
+ }
498
+ throw await this.abortPush(error, transactionManager, run);
499
+ }
500
+ finally {
501
+ await this.ensureTransactionClosed(transactionManager, run.callbacks);
502
+ }
503
+ }
504
+ /** Phases 1, 2 and 2.5. Any failure throws; counted errors only survive a dry run. */
505
+ async runPushPhases(run) {
506
+ const { options, callbacks } = run;
507
+ if (options.verbose) {
508
+ callbacks?.onLog?.('📝 Processing creates and updates...\n');
509
+ }
510
+ // PHASE 1: creates and updates
511
+ const totals = await this.processAllEntityDirectories(run);
512
+ // PHASE 2: deletions in reverse dependency order
513
+ if (run.deletionAudit && totals.errors === 0) {
514
+ const deletionResult = await this.processDeletionsFromAudit(run.deletionAudit, options, callbacks);
515
+ totals.deleted += deletionResult.deleted;
516
+ totals.errors += deletionResult.errors;
517
+ }
518
+ // PHASE 2.5: deferred records (circular dependencies)
519
+ if (this.deferredRecords.length > 0 && totals.errors === 0) {
520
+ const deferredResult = await this.processDeferredRecords(options, callbacks);
521
+ totals.created += deferredResult.created;
522
+ totals.updated += deferredResult.updated;
523
+ totals.errors += deferredResult.errors;
524
+ }
525
+ if (!options.dryRun && totals.errors > 0) {
526
+ throw new Error(`${totals.errors} record error${totals.errors === 1 ? '' : 's'} occurred; the push was not committed.`);
527
+ }
528
+ return totals;
529
+ }
530
+ async commitPushTransaction(transactionManager) {
531
+ try {
532
+ await transactionManager.commitTransaction();
533
+ }
534
+ catch (error) {
535
+ throw new Error(describeCommitFailure(error, this.hostProvider()?.PlatformKey), { cause: error });
536
+ }
537
+ }
538
+ /** Roll back, say truthfully what is left in the database, and wrap the failure. */
539
+ async abortPush(error, transactionManager, run) {
540
+ const { options, callbacks } = run;
541
+ let rolledBack = true;
542
+ if (!options.dryRun) {
543
+ callbacks?.onWarn?.('\n⚠️ Rolling back database transaction due to error...');
544
+ rolledBack = await transactionManager.rollbackTransaction();
545
+ await this.writeFilesWithCommittedRecords(run);
546
+ for (const line of describeRollbackOutcome(rolledBack, this.committedWrites, configManager.getOriginalCwd())) {
547
+ callbacks?.onWarn?.(line);
548
+ }
549
+ }
550
+ return new PushAbortedError({
551
+ modes: [...new Set(this.directoryModes.values())],
552
+ rolledBack,
553
+ committedWrites: [...this.committedWrites],
554
+ totals: { ...this.runningTotals },
555
+ sqlLogPath: this.sqlLogFilePath,
556
+ cause: error,
557
+ });
558
+ }
559
+ /**
560
+ * A file whose write was deferred to Phase 3 — it contains deletions — never reaches that phase
561
+ * when the push fails. In an isolated directory its creates are committed all the same, so write
562
+ * it now and keep it: otherwise the primary keys those rows were given exist only in the
563
+ * database, and the next push creates them a second time.
564
+ */
565
+ async writeFilesWithCommittedRecords(run) {
566
+ const committedFiles = new Set(this.committedWrites.map((w) => w.filePath));
567
+ for (const deferred of this.deferredFileWrites.values()) {
568
+ if (!committedFiles.has(deferred.filePath)) {
569
+ continue;
570
+ }
571
+ try {
572
+ const payload = deferred.isArray ? deferred.records : deferred.records[0];
573
+ await JsonWriteHelper.writeOrderedRecordData(deferred.filePath, payload);
574
+ this.syncMetadataEngine.invalidateCachedFile(deferred.filePath);
575
+ run.fileBackupManager.releaseBackup(deferred.filePath);
576
+ run.callbacks?.onWarn?.(` kept ${path.relative(configManager.getOriginalCwd(), deferred.filePath) || deferred.filePath}: ` +
577
+ `it holds records that are committed`);
578
+ }
579
+ catch (writeError) {
580
+ run.callbacks?.onWarn?.(`Failed to write ${deferred.filePath} after a partial push: ${writeError instanceof Error ? writeError.message : String(writeError)}`);
581
+ }
582
+ }
583
+ }
584
+ /**
585
+ * The database already committed, so the metadata files must keep what they were written with.
586
+ * Drop the file backups so the caller's error path does not restore stale files.
587
+ */
588
+ async failAfterCommit(error, run) {
589
+ await run.fileBackupManager.cleanup();
590
+ const message = error instanceof Error ? error.message : String(error);
591
+ run.callbacks?.onWarn?.(`⚠️ The database changes were committed, but a step after the commit failed: ${message}. ` +
592
+ `Some metadata files or the incremental state may not match the database; run the push again to bring them in line.`);
593
+ return error;
594
+ }
595
+ async ensureTransactionClosed(transactionManager, callbacks) {
596
+ if (!transactionManager.isInTransaction) {
597
+ return;
598
+ }
599
+ const rolledBack = await transactionManager.rollbackTransaction();
600
+ callbacks?.onWarn?.(rolledBack
601
+ ? '⚠️ The push transaction was still open when the push ended, so it was rolled back.'
602
+ : '❌ The push transaction was still open when the push ended, and rolling it back failed.');
603
+ }
604
+ /** Apply incremental checksums and push timestamps. Called only after the push commits. */
605
+ async persistIncrementalState(syncRootDir) {
606
+ if (!this.stateManager) {
607
+ return;
608
+ }
609
+ for (const [relativePath, checksum] of this.pendingChecksums) {
610
+ this.stateManager.setFileChecksum(relativePath, checksum);
611
+ }
612
+ const pushedAt = new Date().toISOString();
613
+ for (const relativeEntityDir of this.pushedEntityDirs) {
614
+ this.stateManager.setLastPushTimestamp(relativeEntityDir, pushedAt);
615
+ }
616
+ await this.stateManager.pruneStaleChecksums(syncRootDir);
617
+ await this.stateManager.save();
618
+ }
619
+ /** PHASE 1 over every entity directory, in order. */
620
+ async processAllEntityDirectories(run) {
621
+ const { entityDirs, options, callbacks } = run;
622
+ const totals = emptyPushTotals();
623
+ for (const [dirIdx, entityDir] of entityDirs.entries()) {
624
+ // "X of N" position prefix — only when there's more than one directory.
625
+ const progressPrefix = entityDirs.length > 1 ? `[${dirIdx + 1}/${entityDirs.length}] ` : '';
626
+ const entityConfig = await loadEntityConfig(entityDir);
627
+ if (!entityConfig) {
628
+ const warning = `Skipping ${entityDir} - no valid entity configuration`;
629
+ this.warnings.push(warning);
630
+ callbacks?.onWarn?.(warning);
631
+ totals.skipped++;
632
+ continue;
633
+ }
634
+ const dirName = path.relative(process.cwd(), entityDir) || '.';
635
+ this.useDirectoryMode(entityDir, options);
636
+ this.announceDirectory(dirName, progressPrefix, entityConfig.entity, entityDir, options, callbacks);
637
+ const result = await this.processEntityDirectory(entityDir, entityConfig, options, run.fileBackupManager, callbacks, run.configDir);
638
+ this.reportDirectoryResult(progressPrefix, dirName, result, options, callbacks);
639
+ addPushTotals(totals, result);
640
+ addPushTotals(this.runningTotals, result);
641
+ }
642
+ return totals;
643
+ }
644
+ announceDirectory(dirName, progressPrefix, entityName, entityDir, options, callbacks) {
645
+ // The folder header is verbose-only; the live spinner still shows "[X/N] Processing <dir>…".
646
+ if (options.verbose) {
647
+ callbacks?.onLog?.(`\n📁 ${dirName}:`);
648
+ }
649
+ if (callbacks?.onProgress) {
650
+ callbacks.onProgress(`${progressPrefix}Processing ${dirName}...`);
651
+ }
652
+ else {
653
+ callbacks?.onLog?.(` ⏳ Processing...`);
654
+ }
655
+ if (options.verbose) {
656
+ callbacks?.onLog?.(`Processing ${entityName} in ${entityDir}`);
657
+ }
658
+ }
659
+ /** One compact line per directory, plus the per-status breakdown in verbose mode. */
660
+ reportDirectoryResult(progressPrefix, dirName, result, options, callbacks) {
661
+ const dirTotal = result.created + result.updated + result.unchanged + result.deleted + result.skipped;
662
+ const { text: dirSummary, changed: dirChanged } = this.formatDirectorySummary(progressPrefix, dirName, result, dirTotal);
663
+ if (callbacks?.onProgress && callbacks?.onSuccess) {
664
+ callbacks.onSuccess(dirSummary, dirChanged);
665
+ }
666
+ else {
667
+ callbacks?.onLog?.(` ${dirSummary}`);
668
+ }
669
+ if (options.verbose && (dirTotal > 0 || result.errors > 0)) {
670
+ this.logDirectoryBreakdown(result, dirTotal, callbacks);
671
+ }
672
+ }
673
+ logDirectoryBreakdown(result, dirTotal, callbacks) {
674
+ const lines = [
675
+ [result.created, ` ✓ Created: ${result.created}`],
676
+ [result.updated, ` ✓ Updated: ${result.updated}`],
677
+ [result.deleted, ` ✓ Deleted: ${result.deleted}`],
678
+ [result.deferred, ` ⏳ Deferred: ${result.deferred}`],
679
+ [result.unchanged, ` - Unchanged: ${result.unchanged}`],
680
+ [result.skipped, ` - Skipped: ${result.skipped}`],
681
+ [result.errors, ` ✗ Errors: ${result.errors}`],
682
+ ];
683
+ callbacks?.onLog?.(` Total processed: ${dirTotal} records`);
684
+ for (const [count, line] of lines) {
685
+ if (count > 0) {
686
+ callbacks?.onLog?.(line);
687
+ }
688
+ }
689
+ }
481
690
  async processEntityDirectory(entityDir, entityConfig, options, fileBackupManager, callbacks, syncRootDir) {
482
691
  let created = 0;
483
692
  let updated = 0;
@@ -563,18 +772,9 @@ export class PushService {
563
772
  if (options.verbose) {
564
773
  callbacks?.onLog?.(` Analyzed ${analysisResult.sortedRecords.length} records (including nested)`);
565
774
  }
566
- // Create batch context for in-memory entity resolution
567
- // Note: While JavaScript is single-threaded, async operations can interleave.
568
- // Map operations themselves are atomic, but we ensure records are added to
569
- // the context AFTER successful save to maintain consistency.
775
+ // Records are added to the batch context only AFTER a successful save (applyProcessResult),
776
+ // so later lookups in this file see consistent state.
570
777
  const batchContext = new BatchContextIndex();
571
- // One provider per JSON-root graph (Action + nested Action Params share a
572
- // connection). Parallelize sibling roots only. Drain a graph when its last
573
- // level finishes, or when TransactionDepth is already 0 (Save settled).
574
- // Peak live independent instances is then the current batch plus any
575
- // leftover-depth graphs still spanning later levels.
576
- const hostProvider = Metadata.Provider; // global-provider-ok: host provider template for GraphProviderPool cloning
577
- const graphPool = new GraphProviderPool(hostProvider, (msg) => callbacks?.onLog?.(msg));
578
778
  const applyProcessResult = (result) => {
579
779
  if (result.batchContextEntry) {
580
780
  batchContext.set(result.batchContextEntry.key, result.batchContextEntry.entity);
@@ -601,83 +801,27 @@ export class PushService {
601
801
  deleted++;
602
802
  else if (result.status === 'skipped')
603
803
  skipped++;
604
- else if (result.status === 'error') {
605
- // A non-throwing record error must not commit leftover graph depth —
606
- // the previous per-record release rolled that work back.
804
+ else if (result.status === 'error')
607
805
  errors++;
608
- graphPool.markFailed();
609
- }
610
806
  else if (result.status === 'deferred') {
611
807
  created++;
612
808
  deferred++;
613
809
  }
614
810
  };
615
- // Fail-fast: the first thrown record error aborts the file. That is
616
- // intentional and matches the parallel path; it is a change from the
617
- // old sequential fallback, which continued after onError.
618
- let runError;
619
- try {
620
- const levels = analysisResult.dependencyLevels && analysisResult.dependencyLevels.length > 0
621
- ? analysisResult.dependencyLevels
622
- : [analysisResult.sortedRecords];
623
- graphPool.noteLevels(levels);
624
- for (let levelIndex = 0; levelIndex < levels.length; levelIndex++) {
625
- const level = levels[levelIndex];
626
- const byGraph = groupRecordsByGraphId(level);
627
- const graphIds = Array.from(byGraph.keys());
628
- const batchSize = options.parallelBatchSize || PARALLEL_BATCH_SIZE;
629
- if (options.verbose && graphIds.length > 1) {
630
- callbacks?.onLog?.(` Level ${levelIndex}: ${level.length} records in ${graphIds.length} graphs (parallel batch ${batchSize})`);
631
- }
632
- for (let i = 0; i < graphIds.length; i += batchSize) {
633
- const batchIds = graphIds.slice(i, i + batchSize);
634
- const batchResults = await Promise.all(batchIds.map(async (graphId) => {
635
- const recs = byGraph.get(graphId);
636
- const provider = await graphPool.obtain(graphId);
637
- const results = [];
638
- for (const flattenedRecord of recs) {
639
- try {
640
- const result = await this.processFlattenedRecord(flattenedRecord, entityDir, options, batchContext, callbacks, entityConfig, true, provider);
641
- results.push({ success: true, result, record: flattenedRecord, graphId });
642
- }
643
- catch (error) {
644
- graphPool.markFailed();
645
- results.push({ success: false, error, record: flattenedRecord, graphId });
646
- break;
647
- }
648
- }
649
- return results;
650
- }));
651
- for (const graphResults of batchResults) {
652
- for (const batchResult of graphResults) {
653
- if (batchResult.success === false) {
654
- const err = batchResult.error;
655
- const rec = batchResult.record;
656
- callbacks?.onLog?.(`\n❌ Processing failed for ${rec.entityName} at ${rec.path}`);
657
- callbacks?.onLog?.(` ${err.message}\n`);
658
- if (err.stack) {
659
- callbacks?.onLog?.(` Stack: ${err.stack}\n`);
660
- }
661
- throw err;
662
- }
663
- applyProcessResult(batchResult.result);
664
- }
665
- }
666
- const drainError = await graphPool.drainBatch(batchIds, levelIndex);
667
- if (drainError)
668
- throw drainError;
669
- }
670
- }
671
- }
672
- catch (e) {
673
- graphPool.markFailed();
674
- runError = e;
675
- }
676
- const settleError = await graphPool.releaseAll();
677
- if (runError)
678
- throw runError;
679
- if (settleError)
680
- throw settleError;
811
+ const levels = analysisResult.dependencyLevels && analysisResult.dependencyLevels.length > 0
812
+ ? analysisResult.dependencyLevels
813
+ : [analysisResult.sortedRecords];
814
+ await this.runFileGraphs({
815
+ filePath,
816
+ entityDir,
817
+ entityConfig,
818
+ options,
819
+ callbacks,
820
+ batchContext,
821
+ levels,
822
+ applyResult: applyProcessResult,
823
+ errorCount: () => errors,
824
+ });
681
825
  // Check if this file has any deletion records (including nested relatedEntities)
682
826
  const hasDeletions = this.hasAnyDeletions(records);
683
827
  // Write back to file (handles both single records and arrays)
@@ -703,14 +847,19 @@ export class PushService {
703
847
  // Drop the cached snapshot — file on disk no longer matches it,
704
848
  // and any later reader within this push must see fresh contents.
705
849
  this.syncMetadataEngine.invalidateCachedFile(filePath);
850
+ // Non-atomic: this file's records are already committed, so a later failure must
851
+ // not restore the old file and lose their primary keys and sync blocks.
852
+ if (this.writeMode === 'isolated') {
853
+ fileBackupManager.releaseBackup(filePath);
854
+ }
706
855
  }
707
856
  }
708
- // Update stored checksum after successful processing (reuse cached value if available)
857
+ // Remember the checksum; it is stored only after the push commits (persistIncrementalState).
709
858
  // Uses resolved content (after @include) so included-file changes are tracked.
710
859
  if (this.stateManager && syncRootDir) {
711
860
  const relativePath = path.relative(syncRootDir, filePath);
712
861
  const checksum = cachedChecksum ?? this.syncEngine.calculateChecksum(fileData);
713
- this.stateManager.setFileChecksum(relativePath, checksum);
862
+ this.pendingChecksums.set(relativePath, checksum);
714
863
  }
715
864
  }
716
865
  catch (fileError) {
@@ -718,15 +867,149 @@ export class PushService {
718
867
  throw fileError;
719
868
  }
720
869
  }
721
- // Persist push timestamp for this entity directory after all files processed
870
+ // The push timestamp for this directory is stored only after the push commits.
722
871
  if (this.stateManager && syncRootDir) {
723
- const relativeEntityDir = path.relative(syncRootDir, entityDir);
724
- this.stateManager.setLastPushTimestamp(relativeEntityDir, new Date().toISOString());
725
- await this.stateManager.pruneStaleChecksums(syncRootDir);
726
- await this.stateManager.save();
872
+ this.pushedEntityDirs.push(path.relative(syncRootDir, entityDir));
727
873
  }
728
874
  return { created, updated, unchanged, deleted, skipped, deferred, errors };
729
875
  }
876
+ /**
877
+ * Run one file's JSON-root graphs, level by level. Atomic mode: one graph at a time on the host.
878
+ * Isolated mode: `graphBatchSize` graphs at once, each on its own independent instance.
879
+ * Fail-fast: the first failed record (thrown or `status: 'error'`) stops the file.
880
+ */
881
+ async runFileGraphs(run) {
882
+ const pendingWrites = new Map();
883
+ const pool = this.createGraphPool(pendingWrites, run.callbacks);
884
+ pool.noteLevels(run.levels);
885
+ let runError;
886
+ try {
887
+ for (let levelIndex = 0; levelIndex < run.levels.length; levelIndex++) {
888
+ await this.runGraphLevel(run, pool, pendingWrites, levelIndex);
889
+ }
890
+ }
891
+ catch (e) {
892
+ pool.markFailed();
893
+ runError = e;
894
+ }
895
+ const settleError = await pool.releaseAll();
896
+ if (runError)
897
+ throw runError;
898
+ if (settleError)
899
+ throw settleError;
900
+ }
901
+ createGraphPool(pendingWrites, callbacks) {
902
+ return new GraphProviderPool(this.hostProvider(), {
903
+ mode: this.writeMode === 'isolated' ? 'independent' : 'host',
904
+ log: (msg) => callbacks?.onLog?.(msg),
905
+ onGraphSettled: (graphId, outcome) => this.recordGraphOutcome(pendingWrites, graphId, outcome),
906
+ });
907
+ }
908
+ /** Non-atomic mode: a graph whose instance committed leaves its writes in the database. */
909
+ recordGraphOutcome(pendingWrites, graphId, outcome) {
910
+ const writes = pendingWrites.get(graphId);
911
+ pendingWrites.delete(graphId);
912
+ if (writes && outcome === 'committed') {
913
+ this.committedWrites.push(...writes);
914
+ }
915
+ }
916
+ async runGraphLevel(run, pool, pendingWrites, levelIndex) {
917
+ const byGraph = groupRecordsByGraphId(run.levels[levelIndex]);
918
+ const graphIds = Array.from(byGraph.keys());
919
+ const batchSize = this.graphBatchSize;
920
+ if (run.options.verbose && graphIds.length > 1) {
921
+ run.callbacks?.onLog?.(` Level ${levelIndex}: ${run.levels[levelIndex].length} records in ${graphIds.length} graphs (batch ${batchSize}, ${this.writeMode})`);
922
+ }
923
+ for (let i = 0; i < graphIds.length; i += batchSize) {
924
+ const batchIds = graphIds.slice(i, i + batchSize);
925
+ const outcomes = await Promise.all(batchIds.map((graphId) => this.runGraph(run, pool, graphId, byGraph.get(graphId) ?? [])));
926
+ this.applyGraphOutcomes(run, pool, pendingWrites, outcomes.flat());
927
+ const drainError = await pool.drainBatch(batchIds, levelIndex);
928
+ if (drainError)
929
+ throw drainError;
930
+ }
931
+ }
932
+ /** One graph's records, in order, on the graph's provider. Stops at the first thrown error. */
933
+ async runGraph(run, pool, graphId, records) {
934
+ const provider = await pool.obtain(graphId);
935
+ const outcomes = [];
936
+ for (const record of records) {
937
+ try {
938
+ const result = await this.processFlattenedRecord(record, run.entityDir, run.options, run.batchContext, run.callbacks, run.entityConfig, true, provider);
939
+ // Read the depth NOW: a save that settled its own scope is already committed in isolated
940
+ // mode, and that is true whatever the rest of the graph goes on to do.
941
+ outcomes.push({ success: true, result, record, graphId, settled: provider.TransactionDepth === 0 });
942
+ }
943
+ catch (error) {
944
+ pool.markFailed();
945
+ outcomes.push({ success: false, error, record, graphId });
946
+ break;
947
+ }
948
+ }
949
+ return outcomes;
950
+ }
951
+ /**
952
+ * Apply a batch's results. Successful writes are tracked first, so a failure still reports
953
+ * what isolated graphs committed. Then the first thrown error, or any counted record
954
+ * error outside a dry run, stops the push.
955
+ */
956
+ applyGraphOutcomes(run, pool, pendingWrites, outcomes) {
957
+ let firstFailure;
958
+ for (const outcome of outcomes) {
959
+ if (outcome.success === false) {
960
+ firstFailure ??= outcome;
961
+ continue;
962
+ }
963
+ run.applyResult(outcome.result);
964
+ if (outcome.result.status === 'error') {
965
+ // A non-throwing record error must not commit leftover graph depth.
966
+ pool.markFailed();
967
+ }
968
+ this.trackPendingWrite(pendingWrites, run.filePath, outcome);
969
+ }
970
+ if (firstFailure) {
971
+ this.throwRecordFailure(firstFailure, run.callbacks);
972
+ }
973
+ const errorCount = run.errorCount();
974
+ if (!run.options.dryRun && errorCount > 0) {
975
+ throw new Error(`${errorCount} record${errorCount === 1 ? '' : 's'} in ${path.basename(run.filePath)} could not be pushed ` +
976
+ `(see the errors above). The push stops at the first failed record.`);
977
+ }
978
+ }
979
+ trackPendingWrite(pendingWrites, filePath, outcome) {
980
+ if (this.writeMode !== 'isolated') {
981
+ return; // shared: nothing commits before the push transaction does
982
+ }
983
+ const status = committedStatusOf(outcome.result);
984
+ if (!status) {
985
+ return;
986
+ }
987
+ const write = {
988
+ filePath,
989
+ entityName: outcome.record.entityName,
990
+ recordPath: outcome.record.path,
991
+ status,
992
+ };
993
+ if (outcome.settled) {
994
+ // Committed as it was saved. Reporting it now means a later rollback of the graph's leftover
995
+ // depth cannot make this write disappear from the report while its row is in the database.
996
+ this.committedWrites.push(write);
997
+ return;
998
+ }
999
+ const writes = pendingWrites.get(outcome.graphId) ?? [];
1000
+ writes.push(write);
1001
+ pendingWrites.set(outcome.graphId, writes);
1002
+ }
1003
+ throwRecordFailure(failure, callbacks) {
1004
+ const err = failure.error instanceof Error ? failure.error : new Error(String(failure.error));
1005
+ const rec = failure.record;
1006
+ callbacks?.onLog?.(`\n❌ Processing failed for ${rec.entityName} at ${rec.path}`);
1007
+ callbacks?.onLog?.(` ${err.message}\n`);
1008
+ if (err.stack) {
1009
+ callbacks?.onLog?.(` Stack: ${err.stack}\n`);
1010
+ }
1011
+ throw err;
1012
+ }
730
1013
  async processFlattenedRecord(flattenedRecord, entityDir, options, batchContext, callbacks, entityConfig, allowDefer = true, recordProvider) {
731
1014
  const metadata = new Metadata(); // global-provider-ok: metadata sync operates on the configured provider only
732
1015
  const { record, entityName, parentContext, id: recordId } = flattenedRecord;
@@ -1227,7 +1510,9 @@ export class PushService {
1227
1510
  message: errorMessage,
1228
1511
  });
1229
1512
  // Throw error to trigger rollback and stop processing
1230
- throw new Error(`Failed to save ${entityName} record at ${flattenedRecord.path}: ${errorMessage}`);
1513
+ const saveFailure = new Error(`Failed to save ${entityName} record at ${flattenedRecord.path}: ${errorMessage}`);
1514
+ this.reportedRecordErrors.add(saveFailure);
1515
+ throw saveFailure;
1231
1516
  }
1232
1517
  // Return batch context entry as a side effect instead of mutating directly.
1233
1518
  // The caller applies this sequentially after Promise.all() resolves, avoiding
@@ -1523,7 +1808,9 @@ export class PushService {
1523
1808
  messages.push(' • No deletions');
1524
1809
  }
1525
1810
  messages.push('');
1526
- messages.push('All operations will occur within a transaction and can be rolled back on error.');
1811
+ for (const line of this.transactionBannerLines()) {
1812
+ messages.push(line);
1813
+ }
1527
1814
  messages.push('');
1528
1815
  messages.push('═'.repeat(80));
1529
1816
  messages.push('');
@@ -1547,6 +1834,19 @@ export class PushService {
1547
1834
  * Audit all deletions across all metadata files
1548
1835
  * This pre-processes all records to identify deletion dependencies and order
1549
1836
  */
1837
+ /** What the deletion confirmation says about rollback. Must match the write mode. */
1838
+ transactionBannerLines() {
1839
+ const isolated = [...this.directoryModes.entries()].filter(([, mode]) => mode === 'isolated');
1840
+ if (isolated.length === 0) {
1841
+ return ['All creates, updates and deletes run in one database transaction. If anything fails, nothing is saved.'];
1842
+ }
1843
+ const names = isolated.map(([dir]) => path.relative(process.cwd(), dir) || dir).join(', ');
1844
+ return [
1845
+ 'Deletes and deferred records run in one database transaction and are rolled back on error.',
1846
+ `These directories use isolated transactions, so each create and update in them is committed as soon as it is`,
1847
+ `saved and is NOT rolled back: ${names}.`,
1848
+ ];
1849
+ }
1550
1850
  async auditAllDeletions(entityDirs, options, callbacks) {
1551
1851
  // OPTIMIZATION: Quick scan to check if ANY deletions exist before doing expensive loading
1552
1852
  let hasAnyDeletions = false;
@@ -1838,11 +2138,23 @@ export class PushService {
1838
2138
  }
1839
2139
  }
1840
2140
  catch (error) {
1841
- const err = error;
2141
+ const err = error instanceof Error ? error : new Error(String(error));
1842
2142
  callbacks?.onError?.(` ✗ Failed to process deferred record: ${entityName} (${recordId})`);
1843
2143
  callbacks?.onError?.(` Error: ${err.message}`);
1844
2144
  callbacks?.onError?.(` Tip: Ensure all referenced records exist or remove the ?allowDefer flag`);
2145
+ if (!this.reportedRecordErrors.has(err)) {
2146
+ callbacks?.onRecordError?.({
2147
+ entityName,
2148
+ path: flattenedRecord.path,
2149
+ primaryKey: recordId,
2150
+ message: `Deferred record could not be resolved: ${err.message}`,
2151
+ });
2152
+ }
1845
2153
  errors++;
2154
+ if (!options.dryRun) {
2155
+ // Same as a thrown Phase 1 error: stop, and let push() roll the transaction back.
2156
+ throw new Error(`Failed to process deferred record ${entityName} (${recordId}): ${err.message}`, { cause: err });
2157
+ }
1846
2158
  }
1847
2159
  }
1848
2160
  // Summary