@memberjunction/metadata-sync 2.111.1 → 2.113.0

package/README.md

@@ -828,13 +828,16 @@ Support environment-specific values:
 - `@env:VARIABLE_NAME`
 - Useful for different environments (dev/staging/prod)
 
-### Automatic JSON Stringification
-When a field value is an array or object, the tool automatically converts it to a JSON string for database storage:
-- Arrays and objects are detected and stringified with pretty formatting (2-space indentation)
-- Maintains clean, readable JSON in source files while storing as strings in database
-- Works seamlessly with all field types that accept text content
+### Automatic JSON Stringification with Reference Processing
 
-Examples:
+When a field value is an array or object, the tool automatically:
+1. **Recursively processes** all `@lookup:`, `@file:`, `@parent:`, `@root:` references inside the object
+2. **Converts to JSON string** with pretty formatting (2-space indentation) for database storage
+3. **Maintains clean structure** in source files while storing as strings in database
+
+This is extremely powerful for JSON-typed fields like `Configuration`, `Settings`, `Metadata`, etc.
+
+#### Basic Example
 ```json
 {
   "fields": {
@@ -846,16 +849,88 @@ Examples:
         "items": [1, 2, 3]
       }
     },
-    "Tags": ["tag1", "tag2", "tag3"],
-    "Metadata": {
-      "created": "2024-01-15",
-      "author": "John Doe"
+    "Tags": ["tag1", "tag2", "tag3"]
+  }
+}
+```
+
+The `Configuration` and `Tags` fields will automatically be converted to JSON strings when pushed to the database.
+
+#### Advanced Example: References Inside JSON Fields
+
+**This is the powerful part** - you can use `@lookup:` and other references INSIDE object-typed fields:
+
+```json
+{
+  "fields": {
+    "Name": "Agent Memory Manager Job",
+    "CronExpression": "0 */15 * * * *",
+    "Configuration": {
+      "AgentID": "@lookup:AI Agents.Name=Memory Manager",
+      "InitialMessage": "Analyze recent conversations",
+      "Settings": {
+        "MaxNotes": 5,
+        "Strategy": "Relevant",
+        "TargetAgentID": "@lookup:AI Agents.Name=Sage"
+      }
     }
   }
 }
 ```
 
-The `Configuration`, `Tags`, and `Metadata` fields will automatically be converted to JSON strings when pushed to the database, while maintaining their structured format in your source files.
+When pushed to the database, this becomes:
+```json
+{
+  "Configuration": "{\"AgentID\":\"actual-uuid-here\",\"InitialMessage\":\"Analyze recent conversations\",\"Settings\":{\"MaxNotes\":5,\"Strategy\":\"Relevant\",\"TargetAgentID\":\"another-uuid-here\"}}"
+}
+```
+
+**Benefits:**
+- ✅ **Human-readable**: Use agent names, not UUIDs in your metadata
+- ✅ **Maintainable**: Changes to entity names don't break references
+- ✅ **Type-safe**: Structured objects in source, properly stringified for DB
+- ✅ **Nested support**: References work at any depth in the object tree
+
+#### Common Use Cases
+
+**Scheduled Job Configuration:**
+```json
+{
+  "Configuration": {
+    "AgentID": "@lookup:AI Agents.Name=Report Generator",
+    "Schedule": "daily",
+    "Recipients": "@lookup:Users.Email=admin@company.com"
+  }
+}
+```
+
+**Action Parameters:**
+```json
+{
+  "DefaultParameters": {
+    "TargetEntityID": "@lookup:Entities.Name=Customers",
+    "TemplateID": "@lookup:Templates.Name=Welcome Email",
+    "Settings": {
+      "SendImmediate": true,
+      "Priority": "High"
+    }
+  }
+}
+```
+
+**AI Configuration:**
+```json
+{
+  "AIConfig": {
+    "PreferredModelID": "@lookup:AI Models.Name=GPT 4.1",
+    "FallbackModels": [
+      "@lookup:AI Models.Name=Claude Sonnet 3.7",
+      "@lookup:AI Models.Name=Gemini Pro"
+    ],
+    "Temperature": 0.7
+  }
+}
+```
 
 ### {@include} References in Files
 Enable content composition within non-JSON files (like .md, .html, .txt) using JSDoc-style include syntax:

package/dist/constants/metadata-keywords.d.ts

@@ -0,0 +1,282 @@
+/**
+ * Centralized metadata keyword constants for MemberJunction MetadataSync package.
+ *
+ * These keywords are special prefixes used in metadata JSON files to reference external
+ * content, perform lookups, access environment variables, and establish hierarchical relationships.
+ *
+ * @module metadata-keywords
+ *
+ * @example
+ * // Using @file: to reference external content
+ * {
+ *   "Prompt": "@file:greeting.prompt.md"
+ * }
+ *
+ * @example
+ * // Using @lookup: to find an entity by field value
+ * {
+ *   "CategoryID": "@lookup:AI Prompt Categories.Name=Examples"
+ * }
+ *
+ * @example
+ * // Using @parent: to reference parent entity fields
+ * {
+ *   "PromptID": "@parent:ID"
+ * }
+ */
+/**
+ * Metadata keyword constants.
+ * These are the special @ prefixes recognized by MetadataSync for field value processing.
+ */
+export declare const METADATA_KEYWORDS: {
+    /**
+     * @file: - Loads content from an external file
+     *
+     * Reads the contents of a file (relative to the JSON metadata file) and uses it as the field value.
+     * Supports text files, markdown, JSON, and other formats.
+     *
+     * @example
+     * "@file:greeting.prompt.md"
+     * "@file:./shared/common-prompt.md"
+     * "@file:../templates/standard-header.md"
+     */
+    readonly FILE: "@file:";
+    /**
+     * @lookup: - Looks up an entity record by field value(s)
+     *
+     * Finds an entity record matching the specified criteria and uses its ID.
+     * Supports single-field and multi-field lookups, with optional auto-creation.
+     *
+     * @example
+     * "@lookup:AI Prompt Types.Name=Chat"
+     * "@lookup:Users.Email=john@example.com&Department=Sales"
+     * "@lookup:Categories.Name=Examples?create"
+     * "@lookup:Categories.Name=Examples?create&Description=Example prompts"
+     */
+    readonly LOOKUP: "@lookup:";
+    /**
+     * @parent: - References a field from the parent entity
+     *
+     * In nested/related entity structures, accesses a field value from the immediate parent record.
+     * Only valid when processing nested entities that have a parent context.
+     *
+     * @example
+     * "@parent:ID"
+     * "@parent:Name"
+     * "@parent:CategoryID"
+     */
+    readonly PARENT: "@parent:";
+    /**
+     * @root: - References a field from the root entity
+     *
+     * In nested/related entity structures, accesses a field value from the top-level root record.
+     * Only valid when processing nested entities that have a root context.
+     *
+     * @example
+     * "@root:ID"
+     * "@root:Name"
+     */
+    readonly ROOT: "@root:";
+    /**
+     * @env: - Reads an environment variable
+     *
+     * Gets the value of an environment variable at runtime.
+     * Useful for configuration values that differ between environments.
+     *
+     * @example
+     * "@env:VARIABLE_NAME"
+     * "@env:NODE_ENV"
+     */
+    readonly ENV: "@env:";
+    /**
+     * @url: - Fetches content from a URL
+     *
+     * Downloads content from a remote URL and uses it as the field value.
+     * Supports HTTP/HTTPS URLs and file:// URLs.
+     *
+     * @example
+     * "@url:https://example.com/prompts/greeting.md"
+     * "@url:https://raw.githubusercontent.com/company/prompts/main/customer.md"
+     */
+    readonly URL: "@url:";
+    /**
+     * @template: - Loads a template JSON file
+     *
+     * Loads a JSON template file and uses its contents, replacing any template variables.
+     * Useful for standardizing common configurations across multiple records.
+     *
+     * @example
+     * "@template:templates/standard-ai-models.json"
+     */
+    readonly TEMPLATE: "@template:";
+    /**
+     * @include or @include.* - Includes content from another file
+     *
+     * Special directive (not a field value) that merges content from external files.
+     * Can be used in both objects and arrays. Supports spread and property modes.
+     *
+     * @example
+     * { "@include": "common-fields.json" }
+     * [ "@include:items.json" ]
+     * { "@include.models": { "file": "models.json", "mode": "property" } }
+     */
+    readonly INCLUDE: "@include";
+};
+/**
+ * Type representing all metadata keyword values.
+ */
+export type MetadataKeyword = typeof METADATA_KEYWORDS[keyof typeof METADATA_KEYWORDS];
+/**
+ * Type representing metadata keyword types (without the colon for keywords that have it).
+ */
+export type MetadataKeywordType = 'file' | 'lookup' | 'parent' | 'root' | 'env' | 'url' | 'template' | 'include';
+/**
+ * Array of all metadata keyword prefixes for iteration.
+ * This array maintains the order of keywords for consistent processing.
+ */
+export declare const METADATA_KEYWORD_PREFIXES: ReadonlyArray<string>;
+/**
+ * Checks if a value is a string that starts with any recognized metadata keyword.
+ *
+ * This is a type-safe check that handles non-string values gracefully.
+ * Returns false for null, undefined, objects, numbers, etc.
+ *
+ * @param value - The value to check (can be any type)
+ * @returns true if value is a string starting with a metadata keyword, false otherwise
+ *
+ * @example
+ * isMetadataKeyword('@file:template.md')  // true
+ * isMetadataKeyword('@lookup:Users.Email=test@example.com')  // true
+ * isMetadataKeyword('@parent:ID')  // true
+ * isMetadataKeyword('regular string')  // false
+ * isMetadataKeyword(123)  // false
+ * isMetadataKeyword(null)  // false
+ * isMetadataKeyword('@unknown:value')  // false
+ */
+export declare function isMetadataKeyword(value: unknown): value is string;
+/**
+ * Determines which metadata keyword type a string value uses.
+ *
+ * Examines the prefix of a string and returns the corresponding keyword type.
+ * Returns null if the value doesn't use any recognized metadata keyword.
+ *
+ * @param value - The string value to analyze
+ * @returns The keyword type ('file', 'lookup', etc.) or null if no keyword is found
+ *
+ * @example
+ * getMetadataKeywordType('@file:template.md')  // 'file'
+ * getMetadataKeywordType('@lookup:Users.Name=John')  // 'lookup'
+ * getMetadataKeywordType('@parent:ID')  // 'parent'
+ * getMetadataKeywordType('@include')  // 'include'
+ * getMetadataKeywordType('regular string')  // null
+ * getMetadataKeywordType('@unknown:value')  // null
+ */
+export declare function getMetadataKeywordType(value: string): MetadataKeywordType | null;
+/**
+ * Type-safe check for metadata keywords that handles any value type.
+ *
+ * This is an alias for isMetadataKeyword() provided for semantic clarity
+ * when you want to explicitly check if a value has a metadata keyword.
+ *
+ * @param value - Any value to check
+ * @returns true if value is a string with a metadata keyword, false otherwise
+ *
+ * @example
+ * const fieldValue = record.Get('SomeField');
+ * if (hasMetadataKeyword(fieldValue)) {
+ *   // fieldValue is guaranteed to be a string with a metadata keyword
+ *   const type = getMetadataKeywordType(fieldValue);
+ * }
+ */
+export declare function hasMetadataKeyword(value: unknown): value is string;
+/**
+ * Checks if a string starts with @ but is NOT a metadata keyword.
+ *
+ * This is useful for filtering out @ strings that are not metadata keywords,
+ * such as npm package names (@mui/material, @angular/core) or email addresses.
+ *
+ * @param value - The value to check
+ * @returns true if value starts with @ but is not a metadata keyword
+ *
+ * @example
+ * isNonKeywordAtSymbol('@mui/material')  // true
+ * isNonKeywordAtSymbol('@angular/core')  // true
+ * isNonKeywordAtSymbol('@file:template.md')  // false
+ * isNonKeywordAtSymbol('regular string')  // false
+ */
+export declare function isNonKeywordAtSymbol(value: unknown): boolean;
+/**
+ * Extracts the value portion after a metadata keyword prefix.
+ *
+ * Removes the keyword prefix from a string and returns the remaining value.
+ * Returns null if the string doesn't use a metadata keyword.
+ *
+ * @param value - The string containing a metadata keyword
+ * @returns The value after the keyword prefix, or null if no keyword found
+ *
+ * @example
+ * extractKeywordValue('@file:template.md')  // 'template.md'
+ * extractKeywordValue('@lookup:Users.Email=test@example.com')  // 'Users.Email=test@example.com'
+ * extractKeywordValue('@parent:ID')  // 'ID'
+ * extractKeywordValue('regular string')  // null
+ */
+export declare function extractKeywordValue(value: string): string | null;
+/**
+ * List of metadata keywords that require a parent context to function.
+ * These keywords cannot be used at the top level of metadata - they only work
+ * in nested/related entities where a parent record exists.
+ */
+export declare const CONTEXT_DEPENDENT_KEYWORDS: readonly ["@parent:", "@root:"];
+/**
+ * List of metadata keywords that reference external resources.
+ * These keywords load content from files, URLs, or other external sources.
+ */
+export declare const EXTERNAL_REFERENCE_KEYWORDS: readonly ["@file:", "@url:", "@template:"];
+/**
+ * List of metadata keywords that perform database lookups.
+ */
+export declare const LOOKUP_KEYWORDS: readonly ["@lookup:"];
+/**
+ * List of metadata keywords that access runtime configuration.
+ */
+export declare const RUNTIME_KEYWORDS: readonly ["@env:"];
+/**
+ * Checks if a keyword requires a parent/root context.
+ *
+ * @param keyword - The keyword to check
+ * @returns true if the keyword requires context (parent or root)
+ *
+ * @example
+ * isContextDependentKeyword('@parent:')  // true
+ * isContextDependentKeyword('@root:')  // true
+ * isContextDependentKeyword('@file:')  // false
+ */
+export declare function isContextDependentKeyword(keyword: string): boolean;
+/**
+ * Checks if a keyword references an external resource.
+ *
+ * @param keyword - The keyword to check
+ * @returns true if the keyword loads external content
+ *
+ * @example
+ * isExternalReferenceKeyword('@file:')  // true
+ * isExternalReferenceKeyword('@url:')  // true
+ * isExternalReferenceKeyword('@lookup:')  // false
+ */
+export declare function isExternalReferenceKeyword(keyword: string): boolean;
+/**
+ * Creates a metadata keyword reference string.
+ *
+ * Helper function to construct properly formatted keyword references.
+ * This ensures consistent formatting across the codebase.
+ *
+ * @param type - The keyword type
+ * @param value - The value after the keyword
+ * @returns Formatted keyword reference string
+ *
+ * @example
+ * createKeywordReference('file', 'template.md')  // '@file:template.md'
+ * createKeywordReference('lookup', 'Users.Email=test@example.com')  // '@lookup:Users.Email=test@example.com'
+ * createKeywordReference('parent', 'ID')  // '@parent:ID'
+ */
+export declare function createKeywordReference(type: MetadataKeywordType, value: string): string;