@memberjunction/metadata-sync 2.53.0 → 2.55.0

package/README.md

@@ -756,6 +756,84 @@ Support environment-specific values:
 - `@env:VARIABLE_NAME`
 - Useful for different environments (dev/staging/prod)
 
+### {@include} References in Files (NEW)
+Enable content composition within non-JSON files (like .md, .html, .txt) using JSDoc-style include syntax:
+- Pattern: `{@include path/to/file.ext}`
+- Supports relative paths from the containing file
+- Recursive includes (includes within includes)
+- Circular reference detection prevents infinite loops
+- Works seamlessly with `@file:` references
+
+#### How It Works
+When a JSON metadata file uses `@file:` to reference an external file, the MetadataSync tool:
+1. Loads the referenced file
+2. Scans for `{@include}` patterns
+3. Recursively resolves all includes
+4. Returns the fully composed content
+
+#### Example Usage
+```markdown
+# My Prompt Template
+
+## System Instructions
+{@include ./shared/system-instructions.md}
+
+## Context
+{@include ../common/context-header.md}
+
+## Task
+Please analyze the following...
+```
+
+#### Complex Example with Nested Includes
+Directory structure:
+```
+prompts/
+├── customer-service/
+│   ├── greeting.json          # Uses @file:greeting.md
+│   ├── greeting.md            # Contains {@include} references
+│   └── shared/
+│       ├── tone.md
+│       └── guidelines.md
+└── common/
+    ├── company-info.md
+    └── legal-disclaimer.md
+```
+
+greeting.json:
+```json
+{
+  "fields": {
+    "Name": "Customer Greeting",
+    "Prompt": "@file:greeting.md"
+  }
+}
+```
+
+greeting.md:
+```markdown
+# Customer Service Greeting
+
+{@include ./shared/tone.md}
+
+## Guidelines
+{@include ./shared/guidelines.md}
+
+## Company Information
+{@include ../common/company-info.md}
+
+## Legal
+{@include ../common/legal-disclaimer.md}
+```
+
+The final content pushed to the database will have all includes fully resolved.
+
+Benefits:
+- **DRY Principle**: Share common content across multiple files
+- **Maintainability**: Update shared content in one place
+- **Flexibility**: Build complex documents from modular parts
+- **Validation**: Automatic checking of included file existence and circular references
+
 ### @template: References (NEW)
 Enable JSON template composition for reusable configurations:
 

package/dist/commands/push/index.js

@@ -515,7 +515,7 @@ class Push extends core_1.Command {
             this.log(`Processing ${jsonFiles.length} records in ${path_1.default.relative(process.cwd(), entityDir) || '.'}`);
         }
         // First, process all JSON files in this directory
-        await this.processJsonFiles(jsonFiles, entityDir, entityConfig, syncEngine, flags, result, fileBackupManager);
+        await this.processJsonFiles(jsonFiles, entityDir, entityConfig, syncEngine, flags, result, fileBackupManager, syncConfig);
         // Then, recursively process subdirectories
         const entries = await fs_extra_1.default.readdir(entityDir, { withFileTypes: true });
         for (const entry of entries) {
@@ -568,7 +568,7 @@ class Push extends core_1.Command {
         }
         return result;
     }
-    async processJsonFiles(jsonFiles, entityDir, entityConfig, syncEngine, flags, result, fileBackupManager) {
+    async processJsonFiles(jsonFiles, entityDir, entityConfig, syncEngine, flags, result, fileBackupManager, syncConfig) {
         if (jsonFiles.length === 0) {
             return;
         }
@@ -595,7 +595,7 @@ class Push extends core_1.Command {
                     const recordData = records[i];
                     // Process the record
                     const recordLineNumber = lineNumbers.get(i); // Get line number for this array index
-                    const pushResult = await this.pushRecord(recordData, entityConfig.entity, path_1.default.dirname(filePath), file, defaults, syncEngine, flags['dry-run'], flags.verbose, isArray ? i : undefined, fileBackupManager, recordLineNumber);
+                    const pushResult = await this.pushRecord(recordData, entityConfig.entity, path_1.default.dirname(filePath), file, defaults, syncEngine, flags['dry-run'], flags.verbose, isArray ? i : undefined, fileBackupManager, recordLineNumber, syncConfig);
                     if (!flags['dry-run']) {
                         // Don't count duplicates in stats
                         if (!pushResult.isDuplicate) {
@@ -643,7 +643,7 @@ class Push extends core_1.Command {
             spinner.stop();
         }
     }
-    async pushRecord(recordData, entityName, baseDir, fileName, defaults, syncEngine, dryRun, verbose = false, arrayIndex, fileBackupManager, lineNumber) {
+    async pushRecord(recordData, entityName, baseDir, fileName, defaults, syncEngine, dryRun, verbose = false, arrayIndex, fileBackupManager, lineNumber, syncConfig) {
         // Load or create entity
         let entity = null;
         let isNew = false;
@@ -654,8 +654,7 @@ class Push extends core_1.Command {
                 const pkDisplay = Object.entries(recordData.primaryKey)
                     .map(([key, value]) => `${key}=${value}`)
                     .join(', ');
-                // Load sync config to check autoCreateMissingRecords setting
-                const syncConfig = await (0, config_1.loadSyncConfig)(config_manager_1.configManager.getOriginalCwd());
+                // Use passed sync config to check autoCreateMissingRecords setting
                 const autoCreate = syncConfig?.push?.autoCreateMissingRecords ?? false;
                 if (!autoCreate) {
                     const fileRef = lineNumber ? `${fileName}:${lineNumber}` : fileName;
@@ -787,7 +786,7 @@ class Push extends core_1.Command {
             const fullFilePath = path_1.default.join(baseDir, fileName);
             relatedStats = await this.processRelatedEntities(recordData.relatedEntities, entity, entity, // root is same as parent for top level
             baseDir, syncEngine, verbose, fileBackupManager, 1, // indentLevel
-            fullFilePath, arrayIndex);
+            fullFilePath, arrayIndex, syncConfig);
         }
         // Update the local file with new primary key if created
         if (isNew) {
@@ -817,7 +816,7 @@ class Push extends core_1.Command {
         }
         return { isNew, wasActuallyUpdated, isDuplicate, relatedStats };
     }
-    async processRelatedEntities(relatedEntities, parentEntity, rootEntity, baseDir, syncEngine, verbose = false, fileBackupManager, indentLevel = 1, parentFilePath, parentArrayIndex) {
+    async processRelatedEntities(relatedEntities, parentEntity, rootEntity, baseDir, syncEngine, verbose = false, fileBackupManager, indentLevel = 1, parentFilePath, parentArrayIndex, syncConfig) {
         const indent = '  '.repeat(indentLevel);
         const stats = { created: 0, updated: 0, unchanged: 0 };
         for (const [entityName, records] of Object.entries(relatedEntities)) {
@@ -836,8 +835,7 @@ class Push extends core_1.Command {
                             const pkDisplay = Object.entries(relatedRecord.primaryKey)
                                 .map(([key, value]) => `${key}=${value}`)
                                 .join(', ');
-                            // Load sync config to check autoCreateMissingRecords setting
-                            const syncConfig = await (0, config_1.loadSyncConfig)(config_manager_1.configManager.getOriginalCwd());
+                            // Use passed sync config to check autoCreateMissingRecords setting
                             const autoCreate = syncConfig?.push?.autoCreateMissingRecords ?? false;
                             if (!autoCreate) {
                                 const fileRef = parentFilePath ? path_1.default.relative(config_manager_1.configManager.getOriginalCwd(), parentFilePath) : 'unknown';
@@ -991,7 +989,7 @@ class Push extends core_1.Command {
                     }
                     // Process nested related entities if any
                     if (relatedRecord.relatedEntities) {
-                        const nestedStats = await this.processRelatedEntities(relatedRecord.relatedEntities, entity, rootEntity, baseDir, syncEngine, verbose, fileBackupManager, indentLevel + 1, parentFilePath, parentArrayIndex);
+                        const nestedStats = await this.processRelatedEntities(relatedRecord.relatedEntities, entity, rootEntity, baseDir, syncEngine, verbose, fileBackupManager, indentLevel + 1, parentFilePath, parentArrayIndex, syncConfig);
                         // Accumulate nested stats
                         stats.created += nestedStats.created;
                         stats.updated += nestedStats.updated;