@memberjunction/metadata-sync 2.112.0 → 2.113.0

package/README.md

@@ -212,7 +212,7 @@ Entity classes are located in:
 
 3. **Runtime Metadata Discovery**:
    ```typescript
-   import { Metadata } from '@memberjunction/global';
+   import { Metadata } from '@memberjunction/core';
    
    const md = new Metadata();
    const entityInfo = md.EntityByName('Templates');
@@ -474,13 +474,13 @@ With `"filePattern": ".*.json"`:
 
 ### Troubleshooting Quick Reference
 
-| Error Message                            | Cause                                      | Solution                                               |
-| ---------------------------------------- | ------------------------------------------ | ------------------------------------------------------ |
-| `No entity directories found`            | Missing .mj-sync.json or wrong filePattern | Check .mj-sync.json exists and uses `"*.json"`         |
-| `Field 'X' does not exist on entity 'Y'` | Using non-existent field                   | Check BaseEntity class in entity_subclasses.ts         |
-| `User ID cannot be null`                 | Missing required UserID                    | Add `"UserID": "ECAFCCEC-6A37-EF11-86D4-000D3A4E707E"` |
-| `Processing 0 records`                   | Files don't match filePattern              | Check files match pattern in .mj-sync.json             |
-| Failed validation                        | Wrong data type or format                  | Check BaseEntity class for field types                 |
+| Error Message | Cause | Solution |
+|---------------|-------|----------|
+| `No entity directories found` | Missing .mj-sync.json or wrong filePattern | Check .mj-sync.json exists and uses `"*.json"` |
+| `Field 'X' does not exist on entity 'Y'` | Using non-existent field | Check BaseEntity class in entity_subclasses.ts |
+| `User ID cannot be null` | Missing required UserID | Add `"UserID": "ECAFCCEC-6A37-EF11-86D4-000D3A4E707E"` |
+| `Processing 0 records` | Files don't match filePattern | Check files match pattern in .mj-sync.json |
+| Failed validation | Wrong data type or format | Check BaseEntity class for field types |
 
 ### System User ID Reference
 
@@ -828,13 +828,16 @@ Support environment-specific values:
 - `@env:VARIABLE_NAME`
 - Useful for different environments (dev/staging/prod)
 
-### Automatic JSON Stringification
-When a field value is an array or object, the tool automatically converts it to a JSON string for database storage:
-- Arrays and objects are detected and stringified with pretty formatting (2-space indentation)
-- Maintains clean, readable JSON in source files while storing as strings in database
-- Works seamlessly with all field types that accept text content
+### Automatic JSON Stringification with Reference Processing
 
-Examples:
+When a field value is an array or object, the tool automatically:
+1. **Recursively processes** all `@lookup:`, `@file:`, `@parent:`, `@root:` references inside the object
+2. **Converts to JSON string** with pretty formatting (2-space indentation) for database storage
+3. **Maintains clean structure** in source files while storing as strings in database
+
+This is extremely powerful for JSON-typed fields like `Configuration`, `Settings`, `Metadata`, etc.
+
+#### Basic Example
 ```json
 {
   "fields": {
@@ -846,16 +849,88 @@ Examples:
         "items": [1, 2, 3]
       }
     },
-    "Tags": ["tag1", "tag2", "tag3"],
-    "Metadata": {
-      "created": "2024-01-15",
-      "author": "John Doe"
+    "Tags": ["tag1", "tag2", "tag3"]
+  }
+}
+```
+
+The `Configuration` and `Tags` fields will automatically be converted to JSON strings when pushed to the database.
+
+#### Advanced Example: References Inside JSON Fields
+
+**This is the powerful part** - you can use `@lookup:` and other references INSIDE object-typed fields:
+
+```json
+{
+  "fields": {
+    "Name": "Agent Memory Manager Job",
+    "CronExpression": "0 */15 * * * *",
+    "Configuration": {
+      "AgentID": "@lookup:AI Agents.Name=Memory Manager",
+      "InitialMessage": "Analyze recent conversations",
+      "Settings": {
+        "MaxNotes": 5,
+        "Strategy": "Relevant",
+        "TargetAgentID": "@lookup:AI Agents.Name=Sage"
+      }
     }
   }
 }
 ```
 
-The `Configuration`, `Tags`, and `Metadata` fields will automatically be converted to JSON strings when pushed to the database, while maintaining their structured format in your source files.
+When pushed to the database, this becomes:
+```json
+{
+  "Configuration": "{\"AgentID\":\"actual-uuid-here\",\"InitialMessage\":\"Analyze recent conversations\",\"Settings\":{\"MaxNotes\":5,\"Strategy\":\"Relevant\",\"TargetAgentID\":\"another-uuid-here\"}}"
+}
+```
+
+**Benefits:**
+- ✅ **Human-readable**: Use agent names, not UUIDs in your metadata
+- ✅ **Maintainable**: Changes to entity names don't break references
+- ✅ **Type-safe**: Structured objects in source, properly stringified for DB
+- ✅ **Nested support**: References work at any depth in the object tree
+
+#### Common Use Cases
+
+**Scheduled Job Configuration:**
+```json
+{
+  "Configuration": {
+    "AgentID": "@lookup:AI Agents.Name=Report Generator",
+    "Schedule": "daily",
+    "Recipients": "@lookup:Users.Email=admin@company.com"
+  }
+}
+```
+
+**Action Parameters:**
+```json
+{
+  "DefaultParameters": {
+    "TargetEntityID": "@lookup:Entities.Name=Customers",
+    "TemplateID": "@lookup:Templates.Name=Welcome Email",
+    "Settings": {
+      "SendImmediate": true,
+      "Priority": "High"
+    }
+  }
+}
+```
+
+**AI Configuration:**
+```json
+{
+  "AIConfig": {
+    "PreferredModelID": "@lookup:AI Models.Name=GPT 4.1",
+    "FallbackModels": [
+      "@lookup:AI Models.Name=Claude Sonnet 3.7",
+      "@lookup:AI Models.Name=Gemini Pro"
+    ],
+    "Temperature": 0.7
+  }
+}
+```
 
 ### {@include} References in Files
 Enable content composition within non-JSON files (like .md, .html, .txt) using JSDoc-style include syntax:
@@ -1510,13 +1585,13 @@ SQL logging is configured in the root-level `.mj-sync.json` file only (not inher
 
 #### SQL Logging Options
 
-| Option              | Type                   | Default         | Description                                                                    |
-| ------------------- | ---------------------- | --------------- | ------------------------------------------------------------------------------ |
-| `enabled`           | boolean                | false           | Whether to enable SQL logging during push operations                           |
-| `outputDirectory`   | string                 | "./sql_logging" | Directory to output SQL log files (relative to command execution directory)    |
-| `formatAsMigration` | boolean                | false           | Whether to format SQL as migration-ready files with Flyway schema placeholders |
-| `filterPatterns`    | string[]               | undefined       | Array of patterns to filter SQL statements (see below)                         |
-| `filterType`        | "exclude" \| "include" | "exclude"       | How to apply filter patterns                                                   |
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `enabled` | boolean | false | Whether to enable SQL logging during push operations |
+| `outputDirectory` | string | "./sql_logging" | Directory to output SQL log files (relative to command execution directory) |
+| `formatAsMigration` | boolean | false | Whether to format SQL as migration-ready files with Flyway schema placeholders |
+| `filterPatterns` | string[] | undefined | Array of patterns to filter SQL statements (see below) |
+| `filterType` | "exclude" \| "include" | "exclude" | How to apply filter patterns |
 
 #### SQL Log File Format
 
@@ -1633,11 +1708,11 @@ Add the `userRoleValidation` configuration to your root `.mj-sync.json` file:
 
 #### Configuration Options
 
-| Option                   | Type     | Default | Description                                   |
-| ------------------------ | -------- | ------- | --------------------------------------------- |
-| `enabled`                | boolean  | false   | Enable user role validation for UserID fields |
-| `allowedRoles`           | string[] | []      | List of role names that are allowed           |
-| `allowUsersWithoutRoles` | boolean  | false   | Allow users without any assigned roles        |
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `enabled` | boolean | false | Enable user role validation for UserID fields |
+| `allowedRoles` | string[] | [] | List of role names that are allowed |
+| `allowUsersWithoutRoles` | boolean | false | Allow users without any assigned roles |
 
 #### How It Works
 
@@ -1864,10 +1939,10 @@ When `recursive: true` is set:
 
 ### Configuration Options
 
-| Option      | Type    | Default | Description                                       |
-| ----------- | ------- | ------- | ------------------------------------------------- |
-| `recursive` | boolean | false   | Enable automatic recursive fetching               |
-| `maxDepth`  | number  | 10      | Maximum recursion depth to prevent infinite loops |
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `recursive` | boolean | false | Enable automatic recursive fetching |
+| `maxDepth` | number | 10 | Maximum recursion depth to prevent infinite loops |
 
 ### Safeguards
 
@@ -1934,24 +2009,24 @@ The pull command now supports smart update capabilities with extensive configura
 
 #### Pull Configuration Options
 
-| Option                        | Type         | Default            | Description                                                                     |
-| ----------------------------- | ------------ | ------------------ | ------------------------------------------------------------------------------- |
-| `filePattern`                 | string       | Entity filePattern | Pattern for finding existing files to update                                    |
-| `createNewFileIfNotFound`     | boolean      | true               | Create files for records not found locally                                      |
-| `newFileName`                 | string       | -                  | Filename for new records when appending (see warning below)                     |
-| `appendRecordsToExistingFile` | boolean      | false              | Append new records to a single file                                             |
-| `updateExistingRecords`       | boolean      | true               | Update existing records found in local files                                    |
-| `preserveFields`              | string[]     | []                 | Fields that retain local values during updates (see detailed explanation below) |
-| `mergeStrategy`               | string       | "merge"            | How to merge updates: "merge", "overwrite", or "skip"                           |
-| `backupBeforeUpdate`          | boolean      | false              | Create timestamped backups before updating files                                |
-| `backupDirectory`             | string       | ".backups"         | Directory name for backup files (relative to entity directory)                  |
-| `filter`                      | string       | -                  | SQL WHERE clause for filtering records                                          |
-| `externalizeFields`           | array/object | -                  | Fields to save as external files with optional patterns                         |
-| `excludeFields`               | string[]     | []                 | Fields to completely omit from pulled data (see detailed explanation below)     |
-| `lookupFields`                | object       | -                  | Foreign keys to convert to @lookup references                                   |
-| `relatedEntities`             | object       | -                  | Related entities to pull as embedded collections                                |
-| `ignoreNullFields`            | boolean      | false              | Exclude fields with null values from pulled data                                |
-| `ignoreVirtualFields`         | boolean      | false              | Exclude virtual fields (view-only fields) from pulled data                      |
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `filePattern` | string | Entity filePattern | Pattern for finding existing files to update |
+| `createNewFileIfNotFound` | boolean | true | Create files for records not found locally |
+| `newFileName` | string | - | Filename for new records when appending (see warning below) |
+| `appendRecordsToExistingFile` | boolean | false | Append new records to a single file |
+| `updateExistingRecords` | boolean | true | Update existing records found in local files |
+| `preserveFields` | string[] | [] | Fields that retain local values during updates (see detailed explanation below) |
+| `mergeStrategy` | string | "merge" | How to merge updates: "merge", "overwrite", or "skip" |
+| `backupBeforeUpdate` | boolean | false | Create timestamped backups before updating files |
+| `backupDirectory` | string | ".backups" | Directory name for backup files (relative to entity directory) |
+| `filter` | string | - | SQL WHERE clause for filtering records |
+| `externalizeFields` | array/object | - | Fields to save as external files with optional patterns |
+| `excludeFields` | string[] | [] | Fields to completely omit from pulled data (see detailed explanation below) |
+| `lookupFields` | object | - | Foreign keys to convert to @lookup references |
+| `relatedEntities` | object | - | Related entities to pull as embedded collections |
+| `ignoreNullFields` | boolean | false | Exclude fields with null values from pulled data |
+| `ignoreVirtualFields` | boolean | false | Exclude virtual fields (view-only fields) from pulled data |
 
 > **⚠️ Important Configuration Warning**
 > 
@@ -2459,14 +2534,14 @@ When using virtual properties, related required fields are skipped:
 
 ### Common Validation Errors
 
-| Error                      | Cause                  | Solution                                   |
-| -------------------------- | ---------------------- | ------------------------------------------ |
-| `Field "X" does not exist` | Typo or wrong entity   | Check entity definition in generated files |
-| `Entity "X" not found`     | Wrong entity name      | Use exact entity name from database        |
-| `File not found`           | Bad @file: reference   | Check file path is relative and exists     |
-| `Lookup not found`         | No matching record     | Verify lookup value or use ?create         |
-| `Circular dependency`      | A→B→A references       | Restructure to avoid cycles                |
-| `Required field missing`   | Missing required field | Add field with appropriate value           |
+| Error | Cause | Solution |
+|-------|-------|----------|
+| `Field "X" does not exist` | Typo or wrong entity | Check entity definition in generated files |
+| `Entity "X" not found` | Wrong entity name | Use exact entity name from database |
+| `File not found` | Bad @file: reference | Check file path is relative and exists |
+| `Lookup not found` | No matching record | Verify lookup value or use ?create |
+| `Circular dependency` | A→B→A references | Restructure to avoid cycles |
+| `Required field missing` | Missing required field | Add field with appropriate value |
 
 ### Validation Configuration
 

package/dist/constants/metadata-keywords.d.ts

@@ -0,0 +1,282 @@
+/**
+ * Centralized metadata keyword constants for MemberJunction MetadataSync package.
+ *
+ * These keywords are special prefixes used in metadata JSON files to reference external
+ * content, perform lookups, access environment variables, and establish hierarchical relationships.
+ *
+ * @module metadata-keywords
+ *
+ * @example
+ * // Using @file: to reference external content
+ * {
+ *   "Prompt": "@file:greeting.prompt.md"
+ * }
+ *
+ * @example
+ * // Using @lookup: to find an entity by field value
+ * {
+ *   "CategoryID": "@lookup:AI Prompt Categories.Name=Examples"
+ * }
+ *
+ * @example
+ * // Using @parent: to reference parent entity fields
+ * {
+ *   "PromptID": "@parent:ID"
+ * }
+ */
+/**
+ * Metadata keyword constants.
+ * These are the special @ prefixes recognized by MetadataSync for field value processing.
+ */
+export declare const METADATA_KEYWORDS: {
+    /**
+     * @file: - Loads content from an external file
+     *
+     * Reads the contents of a file (relative to the JSON metadata file) and uses it as the field value.
+     * Supports text files, markdown, JSON, and other formats.
+     *
+     * @example
+     * "@file:greeting.prompt.md"
+     * "@file:./shared/common-prompt.md"
+     * "@file:../templates/standard-header.md"
+     */
+    readonly FILE: "@file:";
+    /**
+     * @lookup: - Looks up an entity record by field value(s)
+     *
+     * Finds an entity record matching the specified criteria and uses its ID.
+     * Supports single-field and multi-field lookups, with optional auto-creation.
+     *
+     * @example
+     * "@lookup:AI Prompt Types.Name=Chat"
+     * "@lookup:Users.Email=john@example.com&Department=Sales"
+     * "@lookup:Categories.Name=Examples?create"
+     * "@lookup:Categories.Name=Examples?create&Description=Example prompts"
+     */
+    readonly LOOKUP: "@lookup:";
+    /**
+     * @parent: - References a field from the parent entity
+     *
+     * In nested/related entity structures, accesses a field value from the immediate parent record.
+     * Only valid when processing nested entities that have a parent context.
+     *
+     * @example
+     * "@parent:ID"
+     * "@parent:Name"
+     * "@parent:CategoryID"
+     */
+    readonly PARENT: "@parent:";
+    /**
+     * @root: - References a field from the root entity
+     *
+     * In nested/related entity structures, accesses a field value from the top-level root record.
+     * Only valid when processing nested entities that have a root context.
+     *
+     * @example
+     * "@root:ID"
+     * "@root:Name"
+     */
+    readonly ROOT: "@root:";
+    /**
+     * @env: - Reads an environment variable
+     *
+     * Gets the value of an environment variable at runtime.
+     * Useful for configuration values that differ between environments.
+     *
+     * @example
+     * "@env:VARIABLE_NAME"
+     * "@env:NODE_ENV"
+     */
+    readonly ENV: "@env:";
+    /**
+     * @url: - Fetches content from a URL
+     *
+     * Downloads content from a remote URL and uses it as the field value.
+     * Supports HTTP/HTTPS URLs and file:// URLs.
+     *
+     * @example
+     * "@url:https://example.com/prompts/greeting.md"
+     * "@url:https://raw.githubusercontent.com/company/prompts/main/customer.md"
+     */
+    readonly URL: "@url:";
+    /**
+     * @template: - Loads a template JSON file
+     *
+     * Loads a JSON template file and uses its contents, replacing any template variables.
+     * Useful for standardizing common configurations across multiple records.
+     *
+     * @example
+     * "@template:templates/standard-ai-models.json"
+     */
+    readonly TEMPLATE: "@template:";
+    /**
+     * @include or @include.* - Includes content from another file
+     *
+     * Special directive (not a field value) that merges content from external files.
+     * Can be used in both objects and arrays. Supports spread and property modes.
+     *
+     * @example
+     * { "@include": "common-fields.json" }
+     * [ "@include:items.json" ]
+     * { "@include.models": { "file": "models.json", "mode": "property" } }
+     */
+    readonly INCLUDE: "@include";
+};
+/**
+ * Type representing all metadata keyword values.
+ */
+export type MetadataKeyword = typeof METADATA_KEYWORDS[keyof typeof METADATA_KEYWORDS];
+/**
+ * Type representing metadata keyword types (without the colon for keywords that have it).
+ */
+export type MetadataKeywordType = 'file' | 'lookup' | 'parent' | 'root' | 'env' | 'url' | 'template' | 'include';
+/**
+ * Array of all metadata keyword prefixes for iteration.
+ * This array maintains the order of keywords for consistent processing.
+ */
+export declare const METADATA_KEYWORD_PREFIXES: ReadonlyArray<string>;
+/**
+ * Checks if a value is a string that starts with any recognized metadata keyword.
+ *
+ * This is a type-safe check that handles non-string values gracefully.
+ * Returns false for null, undefined, objects, numbers, etc.
+ *
+ * @param value - The value to check (can be any type)
+ * @returns true if value is a string starting with a metadata keyword, false otherwise
+ *
+ * @example
+ * isMetadataKeyword('@file:template.md')  // true
+ * isMetadataKeyword('@lookup:Users.Email=test@example.com')  // true
+ * isMetadataKeyword('@parent:ID')  // true
+ * isMetadataKeyword('regular string')  // false
+ * isMetadataKeyword(123)  // false
+ * isMetadataKeyword(null)  // false
+ * isMetadataKeyword('@unknown:value')  // false
+ */
+export declare function isMetadataKeyword(value: unknown): value is string;
+/**
+ * Determines which metadata keyword type a string value uses.
+ *
+ * Examines the prefix of a string and returns the corresponding keyword type.
+ * Returns null if the value doesn't use any recognized metadata keyword.
+ *
+ * @param value - The string value to analyze
+ * @returns The keyword type ('file', 'lookup', etc.) or null if no keyword is found
+ *
+ * @example
+ * getMetadataKeywordType('@file:template.md')  // 'file'
+ * getMetadataKeywordType('@lookup:Users.Name=John')  // 'lookup'
+ * getMetadataKeywordType('@parent:ID')  // 'parent'
+ * getMetadataKeywordType('@include')  // 'include'
+ * getMetadataKeywordType('regular string')  // null
+ * getMetadataKeywordType('@unknown:value')  // null
+ */
+export declare function getMetadataKeywordType(value: string): MetadataKeywordType | null;
+/**
+ * Type-safe check for metadata keywords that handles any value type.
+ *
+ * This is an alias for isMetadataKeyword() provided for semantic clarity
+ * when you want to explicitly check if a value has a metadata keyword.
+ *
+ * @param value - Any value to check
+ * @returns true if value is a string with a metadata keyword, false otherwise
+ *
+ * @example
+ * const fieldValue = record.Get('SomeField');
+ * if (hasMetadataKeyword(fieldValue)) {
+ *   // fieldValue is guaranteed to be a string with a metadata keyword
+ *   const type = getMetadataKeywordType(fieldValue);
+ * }
+ */
+export declare function hasMetadataKeyword(value: unknown): value is string;
+/**
+ * Checks if a string starts with @ but is NOT a metadata keyword.
+ *
+ * This is useful for filtering out @ strings that are not metadata keywords,
+ * such as npm package names (@mui/material, @angular/core) or email addresses.
+ *
+ * @param value - The value to check
+ * @returns true if value starts with @ but is not a metadata keyword
+ *
+ * @example
+ * isNonKeywordAtSymbol('@mui/material')  // true
+ * isNonKeywordAtSymbol('@angular/core')  // true
+ * isNonKeywordAtSymbol('@file:template.md')  // false
+ * isNonKeywordAtSymbol('regular string')  // false
+ */
+export declare function isNonKeywordAtSymbol(value: unknown): boolean;
+/**
+ * Extracts the value portion after a metadata keyword prefix.
+ *
+ * Removes the keyword prefix from a string and returns the remaining value.
+ * Returns null if the string doesn't use a metadata keyword.
+ *
+ * @param value - The string containing a metadata keyword
+ * @returns The value after the keyword prefix, or null if no keyword found
+ *
+ * @example
+ * extractKeywordValue('@file:template.md')  // 'template.md'
+ * extractKeywordValue('@lookup:Users.Email=test@example.com')  // 'Users.Email=test@example.com'
+ * extractKeywordValue('@parent:ID')  // 'ID'
+ * extractKeywordValue('regular string')  // null
+ */
+export declare function extractKeywordValue(value: string): string | null;
+/**
+ * List of metadata keywords that require a parent context to function.
+ * These keywords cannot be used at the top level of metadata - they only work
+ * in nested/related entities where a parent record exists.
+ */
+export declare const CONTEXT_DEPENDENT_KEYWORDS: readonly ["@parent:", "@root:"];
+/**
+ * List of metadata keywords that reference external resources.
+ * These keywords load content from files, URLs, or other external sources.
+ */
+export declare const EXTERNAL_REFERENCE_KEYWORDS: readonly ["@file:", "@url:", "@template:"];
+/**
+ * List of metadata keywords that perform database lookups.
+ */
+export declare const LOOKUP_KEYWORDS: readonly ["@lookup:"];
+/**
+ * List of metadata keywords that access runtime configuration.
+ */
+export declare const RUNTIME_KEYWORDS: readonly ["@env:"];
+/**
+ * Checks if a keyword requires a parent/root context.
+ *
+ * @param keyword - The keyword to check
+ * @returns true if the keyword requires context (parent or root)
+ *
+ * @example
+ * isContextDependentKeyword('@parent:')  // true
+ * isContextDependentKeyword('@root:')  // true
+ * isContextDependentKeyword('@file:')  // false
+ */
+export declare function isContextDependentKeyword(keyword: string): boolean;
+/**
+ * Checks if a keyword references an external resource.
+ *
+ * @param keyword - The keyword to check
+ * @returns true if the keyword loads external content
+ *
+ * @example
+ * isExternalReferenceKeyword('@file:')  // true
+ * isExternalReferenceKeyword('@url:')  // true
+ * isExternalReferenceKeyword('@lookup:')  // false
+ */
+export declare function isExternalReferenceKeyword(keyword: string): boolean;
+/**
+ * Creates a metadata keyword reference string.
+ *
+ * Helper function to construct properly formatted keyword references.
+ * This ensures consistent formatting across the codebase.
+ *
+ * @param type - The keyword type
+ * @param value - The value after the keyword
+ * @returns Formatted keyword reference string
+ *
+ * @example
+ * createKeywordReference('file', 'template.md')  // '@file:template.md'
+ * createKeywordReference('lookup', 'Users.Email=test@example.com')  // '@lookup:Users.Email=test@example.com'
+ * createKeywordReference('parent', 'ID')  // '@parent:ID'
+ */
+export declare function createKeywordReference(type: MetadataKeywordType, value: string): string;