@aifabrix/builder 2.8.0 → 2.9.0

package/lib/schema/external-datasource.schema.json

@@ -2,17 +2,17 @@
    "$schema":"https://json-schema.org/draft/2020-12/schema",
    "$id":"https://raw.githubusercontent.com/esystemsdev/aifabrix-builder/refs/heads/main/lib/schema/external-datasource.schema.json",
    "title":"External Data Source",
-   "description":"Configuration for AI Fabrix ExternalDataSource entities. Includes metadata schema, ABAC access fields, transformation mappings, OpenAPI/MCP exposure, and sync behavior.",
+   "description":"Configuration for AI Fabrix ExternalDataSource entities. Includes metadata schema, ABAC access fields, transformation mappings, OpenAPI/MCP exposure, execution logic, and sync behavior.",
    "metadata":{
       "key":"external-datasource-schema",
       "name":"External Data Source Configuration Schema",
       "description":"JSON schema for validating ExternalDataSource configuration files",
-      "version":"1.0.0",
+      "version":"1.1.0",
       "type":"schema",
       "category":"integration",
       "author":"AI Fabrix Team",
       "createdAt":"2024-01-01T00:00:00Z",
-      "updatedAt":"2024-01-01T00:00:00Z",
+      "updatedAt":"2025-01-01T00:00:00Z",
       "compatibility":{
          "minVersion":"1.0.0",
          "maxVersion":"2.0.0",
@@ -23,7 +23,9 @@
          "external-datasource",
          "dataplane",
          "integration",
-         "validation"
+         "validation",
+         "cip",
+         "execution"
       ],
       "dependencies":[
          "external-system.schema.json"
@@ -42,6 +44,19 @@
                "Added sync configuration and validation rules"
             ],
             "breaking":false
+         },
+         {
+            "version":"1.1.0",
+            "date":"2025-01-01T00:00:00Z",
+            "changes":[
+               "Added execution.engine with CIP and Python execution modes",
+               "Introduced CIP definition ($defs.cipDefinition and step types)",
+               "Added capabilities for list/get/create/update/delete",
+               "Extended fieldMappings with semantic metadata for AI",
+               "Added exposure profiles for different AI/analytics views",
+               "Added data quality rules, indexing, and AI context hints"
+            ],
+            "breaking":false
          }
       ]
    },
@@ -82,7 +97,13 @@
       },
       "resourceType":{
          "type":"string",
-         "enum":["customer","contact","person","document","deal"],
+         "enum":[
+            "customer",
+            "contact",
+            "person",
+            "document",
+            "deal"
+         ],
          "default":"document",
          "description":"Resource type for the datasource. Maps to database ResourceType enum."
       },
@@ -141,10 +162,44 @@
                         ]
                      },
                      "description":{
-                        "type":"string"
+                        "type":"string",
+                        "description":"Technical description of the normalized field."
                      },
                      "required":{
                         "type":"boolean"
+                     },
+                     "semantic":{
+                        "type":"object",
+                        "description":"Semantic metadata for AI agents and schema generation.",
+                        "properties":{
+                           "title":{
+                              "type":"string",
+                              "description":"Human-friendly label for this field."
+                           },
+                           "description":{
+                              "type":"string",
+                              "description":"Business-level description of the field and how it is used."
+                           },
+                           "example":{
+                              "description":"Example value for this field.",
+                              "type":["string","number","boolean","object","array","null"]
+                           },
+                           "categories":{
+                              "type":"array",
+                              "description":"Logical categories/tags (e.g. ['sales', 'revenue']).",
+                              "items":{
+                                 "type":"string"
+                              }
+                           },
+                           "synonyms":{
+                              "type":"array",
+                              "description":"Optional list of synonyms used in natural language (e.g. 'opportunity', 'sales-case').",
+                              "items":{
+                                 "type":"string"
+                              }
+                           }
+                        },
+                        "additionalProperties":false
                      }
                   }
                }
@@ -199,6 +254,19 @@
             "description":{
                "type":"string",
                "description":"Human-readable description of the exposure model for this data source."
+            },
+            "profiles":{
+               "type":"object",
+               "description":"Named exposure profiles mapping profile keys to lists of normalized field names. Useful for AI-light, AI-embedding, analytics, etc.",
+               "additionalProperties":{
+                  "type":"array",
+                  "items":{
+                     "type":"string",
+                     "pattern":"^[a-zA-Z0-9_]+$"
+                  },
+                  "minItems":1,
+                  "uniqueItems":true
+               }
             }
          },
          "required":[
@@ -238,7 +306,7 @@
       },
       "openapi":{
          "type":"object",
-         "description":"OpenAPI-driven connector configuration.",
+         "description":"OpenAPI-driven connector configuration. Only includes endpoints selected during wizard onboarding. To add more endpoints, run the wizard again.",
          "properties":{
             "enabled":{
                "type":"boolean",
@@ -256,6 +324,7 @@
             },
             "operations":{
                "type":"object",
+               "description":"Selected operations from the OpenAPI document. Only endpoints chosen during wizard onboarding are included.",
                "properties":{
                   "list":{
                      "type":"object",
@@ -361,48 +430,6 @@
             }
          }
       },
-      "mcp":{
-         "type":"object",
-         "description":"Model Context Protocol exposure.",
-         "properties":{
-            "enabled":{
-               "type":"boolean",
-               "default":false
-            },
-            "toolName":{
-               "type":"string",
-               "pattern":"^[a-z0-9-]+$"
-            },
-            "resourceName":{
-               "type":"string",
-               "pattern":"^[a-z0-9-:]+$"
-            },
-            "operations":{
-               "type":"array",
-               "items":{
-                  "type":"object",
-                  "required":[
-                     "name",
-                     "permission"
-                  ],
-                  "properties":{
-                     "name":{
-                        "type":"string",
-                        "pattern":"^[a-z0-9-]+$"
-                     },
-                     "permission":{
-                        "type":"string",
-                        "description":"Must match securityModel.permissions",
-                        "pattern":"^[a-z0-9-:]+$"
-                     },
-                     "description":{
-                        "type":"string"
-                     }
-                  }
-               }
-            }
-         }
-      },
       "validation":{
          "type":"object",
          "description":"Advanced validation rules applied before saving normalized metadata.",
@@ -447,6 +474,252 @@
             }
          }
       },
+      "quality":{
+         "type":"object",
+         "description":"Data quality rules applied after mapping but before persistence and AI exposure.",
+         "properties":{
+            "rejectIf":{
+               "type":"array",
+               "description":"List of conditions that cause a record to be rejected.",
+               "items":{
+                  "type":"object",
+                  "required":[
+                     "field",
+                     "operator"
+                  ],
+                  "properties":{
+                     "field":{
+                        "type":"string",
+                        "pattern":"^[a-zA-Z0-9_]+$",
+                        "description":"Normalized field name to evaluate."
+                     },
+                     "operator":{
+                        "type":"string",
+                        "enum":[
+                           "missing",
+                           "null",
+                           "empty",
+                           "invalidRef",
+                           "lessThan",
+                           "greaterThan",
+                           "equal",
+                           "notEqual"
+                        ]
+                     },
+                     "value":{
+                        "description":"Comparison value for applicable operators.",
+                        "type":["string","number","boolean","null"]
+                     },
+                     "message":{
+                        "type":"string",
+                        "description":"Optional human-readable reason for rejection."
+                     }
+                  },
+                  "additionalProperties":false
+               }
+            }
+         },
+         "additionalProperties":false
+      },
+      "indexing":{
+         "type":"object",
+         "description":"Indexing and embedding strategy for this datasource.",
+         "properties":{
+            "embedding":{
+               "type":"array",
+               "description":"List of normalized fields whose values should be concatenated and embedded for vector search.",
+               "items":{
+                  "type":"string",
+                  "pattern":"^[a-zA-Z0-9_]+$"
+               },
+               "minItems":1,
+               "uniqueItems":true
+            },
+            "uniqueKey":{
+               "type":"string",
+               "pattern":"^[a-zA-Z0-9_]+$",
+               "description":"Normalized field used as unique identifier for deduplication and upserts."
+            },
+            "dedupeStrategy":{
+               "type":"string",
+               "enum":[
+                  "reject",
+                  "latest-wins",
+                  "first-wins",
+                  "merge"
+               ],
+               "default":"latest-wins",
+               "description":"Strategy to apply when multiple records share the same uniqueKey."
+            }
+         },
+         "additionalProperties":false
+      },
+      "context":{
+         "type":"object",
+         "description":"Natural-language and semantic hints for AI agents.",
+         "properties":{
+            "semanticTags":{
+               "type":"array",
+               "description":"High-level tags describing this datasource (e.g. ['crm', 'customer-data', 'deal']).",
+               "items":{
+                  "type":"string"
+               }
+            },
+            "synonyms":{
+               "type":"array",
+               "description":"Terms users might use in natural language to refer to this datasource or its primary entity.",
+               "items":{
+                  "type":"string"
+               }
+            },
+            "naturalLanguageHints":{
+               "type":"array",
+               "description":"Free-form hints for LLMs on how to interpret or use this datasource.",
+               "items":{
+                  "type":"string"
+               }
+            }
+         },
+         "additionalProperties":false
+      },
+      "documentStorage":{
+         "type":"object",
+         "description":"Document storage configuration (optional, enables vector storage)",
+         "properties":{
+            "enabled":{
+               "type":"boolean",
+               "default":true
+            },
+            "flowiseInstanceId":{
+               "type":"string"
+            },
+            "twoPhaseSync":{
+               "type":"boolean",
+               "default":true
+            },
+            "binaryOperation":{
+               "type":"object",
+               "description":"Configuration for binary document retrieval. Supports CIP operations or direct HTTP/OpenAPI.",
+               "oneOf":[
+                  {
+                     "required":[
+                        "cipOperationRef",
+                        "responseType"
+                     ],
+                     "properties":{
+                        "cipOperationRef":{
+                           "type":"string",
+                           "description":"Reference to CIP operation (e.g., 'get' or custom operation name)"
+                        },
+                        "responseType":{
+                           "type":"string",
+                           "enum":[
+                              "binary",
+                              "base64",
+                              "json"
+                           ],
+                           "default":"binary"
+                        },
+                        "binaryField":{
+                           "type":"string"
+                        },
+                        "parameterMapping":{
+                           "type":"object",
+                           "additionalProperties":{
+                              "type":"string"
+                           }
+                        }
+                     },
+                     "additionalProperties":false
+                  },
+                  {
+                     "required":[
+                        "path",
+                        "responseType"
+                     ],
+                     "properties":{
+                        "operationId":{
+                           "type":"string"
+                        },
+                        "method":{
+                           "type":"string",
+                           "enum":[
+                              "GET",
+                              "POST",
+                              "PUT"
+                           ],
+                           "default":"GET"
+                        },
+                        "path":{
+                           "type":"string"
+                        },
+                        "queryParameters":{
+                           "type":"object",
+                           "additionalProperties":{
+                              "type":"string"
+                           }
+                        },
+                        "headers":{
+                           "type":"object",
+                           "additionalProperties":{
+                              "type":"string"
+                           }
+                        },
+                        "requestBody":{
+                           "type":"object"
+                        },
+                        "responseType":{
+                           "type":"string",
+                           "enum":[
+                              "binary",
+                              "base64",
+                              "json"
+                           ],
+                           "default":"binary"
+                        },
+                        "binaryField":{
+                           "type":"string"
+                        },
+                        "parameterMapping":{
+                           "type":"object",
+                           "additionalProperties":{
+                              "type":"string"
+                           }
+                        }
+                     },
+                     "additionalProperties":false
+                  }
+               ]
+            },
+            "processing":{
+               "type":"object",
+               "properties":{
+                  "fileStoragePath":{
+                     "type":"string",
+                     "default":"/data/documents"
+                  },
+                  "aiValidation":{
+                     "type":"object"
+                  },
+                  "spacyEnrichment":{
+                     "type":"object"
+                  },
+                  "notifications":{
+                     "type":"object"
+                  }
+               },
+               "additionalProperties":false
+            },
+            "credentialId":{
+               "type":"string"
+            }
+         },
+         "required":[
+            "flowiseInstanceId",
+            "binaryOperation"
+         ],
+         "additionalProperties":false
+      },
       "portalInput":{
          "type":"array",
          "description":"Optional UI metadata definition for the AI Fabrix portal.",
@@ -506,6 +779,507 @@
                }
             }
          }
+      },
+      "testPayload":{
+         "type":"object",
+         "description":"Test payload configuration for unit and integration testing",
+         "properties":{
+            "payloadTemplate":{
+               "type":"object",
+               "description":"Sample payload matching the expected API response structure. Used for testing field mappings and metadata schema validation.",
+               "additionalProperties":true
+            },
+            "expectedResult":{
+               "type":"object",
+               "description":"Expected normalized result after field mapping transformations (optional, for validation)",
+               "additionalProperties":true
+            }
+         },
+         "additionalProperties":false
+      },
+      "capabilities":{
+         "type":"object",
+         "description":"Declares which logical operations are supported by this datasource.",
+         "properties":{
+            "list":{
+               "type":"boolean",
+               "default":true
+            },
+            "get":{
+               "type":"boolean",
+               "default":false
+            },
+            "create":{
+               "type":"boolean",
+               "default":false
+            },
+            "update":{
+               "type":"boolean",
+               "default":false
+            },
+            "delete":{
+               "type":"boolean",
+               "default":false
+            }
+         },
+         "additionalProperties":false
+      },
+      "execution":{
+         "type":"object",
+         "description":"Execution engine configuration for this datasource (CIP or Python). CIP is the native declarative mode.",
+         "required":[
+            "engine"
+         ],
+         "properties":{
+            "engine":{
+               "type":"string",
+               "enum":[
+                  "cip",
+                  "python"
+               ],
+               "description":"Execution engine. 'cip' for declarative pipelines, 'python' for custom handlers."
+            },
+            "cip":{
+               "$ref":"#/$defs/cipDefinition"
+            },
+            "python":{
+               "$ref":"#/$defs/pythonExecution"
+            }
+         },
+         "additionalProperties":false
+      }
+   },
+   "$defs":{
+      "pythonExecution":{
+         "type":"object",
+         "description":"Python-based execution config for advanced/custom use cases.",
+         "properties":{
+            "entrypoint":{
+               "type":"string",
+               "description":"Python callable in module notation, e.g. 'handlers.hubspot_deal.list_records'."
+            },
+            "codeRef":{
+               "type":"string",
+               "description":"Optional reference to external code bundle identifier or embedded block key."
+            }
+         },
+         "required":[
+            "entrypoint"
+         ],
+         "additionalProperties":false
+      },
+      "cipLineage":{
+         "type":"object",
+         "description":"Optional lineage metadata for a CIP operation or step.",
+         "properties":{
+            "producer":{
+               "type":"string",
+               "description":"Upstream system or connector name (e.g. 'hubspot')."
+            },
+            "entity":{
+               "type":"string",
+               "description":"Upstream entity or resource (e.g. 'deal')."
+            },
+            "operation":{
+               "type":"string",
+               "description":"Logical operation (e.g. 'list', 'get', 'sync-job-1')."
+            }
+         },
+         "additionalProperties":false
+      },
+      "cipDefinition":{
+         "type":"object",
+         "description":"Composable Integration Pipeline (CIP) definition for this datasource.",
+         "required":[
+            "version",
+            "operations"
+         ],
+         "properties":{
+            "version":{
+               "type":"string",
+               "enum":[
+                  "1.0"
+               ]
+            },
+            "inputType":{
+               "type":"string",
+               "const":"void",
+               "description":"Logical input type of the pipeline. Reserved for future extensions."
+            },
+            "outputType":{
+               "type":"string",
+               "const":"records",
+               "description":"Logical output type of the pipeline."
+            },
+            "operations":{
+               "type":"object",
+               "description":"Logical operations (list/get/create/update/delete) implemented for this datasource.",
+               "properties":{
+                  "list":{
+                     "$ref":"#/$defs/cipOperation"
+                  },
+                  "get":{
+                     "$ref":"#/$defs/cipOperation"
+                  },
+                  "create":{
+                     "$ref":"#/$defs/cipOperation"
+                  },
+                  "update":{
+                     "$ref":"#/$defs/cipOperation"
+                  },
+                  "delete":{
+                     "$ref":"#/$defs/cipOperation"
+                  }
+               },
+               "additionalProperties":false
+            }
+         },
+         "additionalProperties":false
+      },
+      "cipOperation":{
+         "type":"object",
+         "required":[
+            "steps"
+         ],
+         "properties":{
+            "description":{
+               "type":"string"
+            },
+            "enabled":{
+               "type":"boolean",
+               "default":true
+            },
+            "lineage":{
+               "$ref":"#/$defs/cipLineage"
+            },
+            "steps":{
+               "type":"array",
+               "minItems":1,
+               "items":{
+                  "$ref":"#/$defs/cipStep"
+               }
+            }
+         },
+         "additionalProperties":false
+      },
+      "cipStep":{
+         "description":"One step in the CIP pipeline.",
+         "oneOf":[
+            {
+               "$ref":"#/$defs/cipStepFetch"
+            },
+            {
+               "$ref":"#/$defs/cipStepPaginate"
+            },
+            {
+               "$ref":"#/$defs/cipStepMap"
+            },
+            {
+               "$ref":"#/$defs/cipStepFilter"
+            },
+            {
+               "$ref":"#/$defs/cipStepOutput"
+            },
+            {
+               "$ref":"#/$defs/cipStepPythonInline"
+            }
+         ]
+      },
+      "cipStepFetch":{
+         "type":"object",
+         "required":[
+            "fetch"
+         ],
+         "properties":{
+            "fetch":{
+               "type":"object",
+               "description":"Fetch raw records from an upstream system.",
+               "required":[
+                  "source"
+               ],
+               "properties":{
+                  "source":{
+                     "type":"string",
+                     "enum":[
+                        "openapi",
+                        "http"
+                     ],
+                     "description":"If 'openapi', refer to openapi.operations.*. If 'http', use explicit method/path."
+                  },
+                  "openapiRef":{
+                     "type":"string",
+                     "enum":[
+                        "list",
+                        "get",
+                        "create",
+                        "update",
+                        "delete"
+                     ],
+                     "description":"Reference to openapi.operations.<openapiRef> entry."
+                  },
+                  "operationId":{
+                     "type":"string",
+                     "description":"OpenAPI operationId to call when source='openapi'. Optional when openapiRef is used."
+                  },
+                  "method":{
+                     "type":"string",
+                     "enum":[
+                        "GET",
+                        "POST"
+                     ],
+                     "description":"Explicit HTTP method when source='http'."
+                  },
+                  "path":{
+                     "type":"string",
+                     "description":"Explicit HTTP path when source='http'."
+                  },
+                  "query":{
+                     "type":"object",
+                     "description":"Static query parameters. ABAC/runtime can still override or append at runtime.",
+                     "additionalProperties":{
+                        "type":[
+                           "string",
+                           "number",
+                           "boolean",
+                           "null"
+                        ]
+                     }
+                  },
+                  "bodyTemplate":{
+                     "description":"Optional static request body or template for POST/PUT fetches.",
+                     "type":["object","string","null"]
+                  }
+               },
+               "allOf":[
+                  {
+                     "if":{
+                        "properties":{
+                           "source":{
+                              "const":"openapi"
+                           }
+                        }
+                     },
+                     "then":{
+                        "anyOf":[
+                           {
+                              "required":[
+                                 "openapiRef"
+                              ]
+                           },
+                           {
+                              "required":[
+                                 "operationId"
+                              ]
+                           }
+                        ]
+                     }
+                  },
+                  {
+                     "if":{
+                        "properties":{
+                           "source":{
+                              "const":"http"
+                           }
+                        }
+                     },
+                     "then":{
+                        "required":[
+                           "method",
+                           "path"
+                        ]
+                     }
+                  }
+               ],
+               "additionalProperties":false
+            },
+            "lineage":{
+               "$ref":"#/$defs/cipLineage"
+            }
+         },
+         "additionalProperties":false
+      },
+      "cipStepPaginate":{
+         "type":"object",
+         "required":[
+            "paginate"
+         ],
+         "properties":{
+            "paginate":{
+               "type":"object",
+               "description":"Pagination strategy applied to results from the preceding fetch.",
+               "required":[
+                  "strategy"
+               ],
+               "properties":{
+                  "strategy":{
+                     "type":"string",
+                     "enum":[
+                        "cursor",
+                        "page",
+                        "offset",
+                        "none"
+                     ]
+                  },
+                  "cursorField":{
+                     "type":"string",
+                     "description":"Field in the response payload containing the cursor value (for 'cursor' strategy)."
+                  },
+                  "cursorParam":{
+                     "type":"string",
+                     "description":"Name of the query parameter used to send the cursor."
+                  },
+                  "pageParam":{
+                     "type":"string",
+                     "description":"Name of the query parameter used to send the page number (for 'page' strategy)."
+                  },
+                  "pageSizeParam":{
+                     "type":"string",
+                     "description":"Name of the query parameter used to send the page size."
+                  },
+                  "pageSize":{
+                     "type":"integer",
+                     "minimum":1,
+                     "description":"Default page size."
+                  },
+                  "maxPages":{
+                     "type":"integer",
+                     "minimum":1,
+                     "description":"Maximum number of pages to fetch."
+                  }
+               },
+               "additionalProperties":false
+            },
+            "lineage":{
+               "$ref":"#/$defs/cipLineage"
+            }
+         },
+         "additionalProperties":false
+      },
+      "cipStepMap":{
+         "type":"object",
+         "required":[
+            "map"
+         ],
+         "properties":{
+            "map":{
+               "type":"object",
+               "description":"Transform raw API records into the normalized metadata model.",
+               "properties":{
+                  "inputPath":{
+                     "type":"string",
+                     "default":"$.items[*]",
+                     "description":"JSONPath-like expression to iterate over records in the upstream payload."
+                  },
+                  "useFieldMappings":{
+                     "type":"boolean",
+                     "default":true,
+                     "description":"If true, apply the datasource.fieldMappings.fields expressions; no inline mapping needed."
+                  },
+                  "inline":{
+                     "type":"object",
+                     "description":"Optional inline mapping overrides { normalizedField -> expression }.",
+                     "additionalProperties":{
+                        "type":"string",
+                        "description":"Pipe-based DSL expression overriding fieldMappings expression."
+                     }
+                  }
+               },
+               "additionalProperties":false
+            },
+            "lineage":{
+               "$ref":"#/$defs/cipLineage"
+            }
+         },
+         "additionalProperties":false
+      },
+      "cipStepFilter":{
+         "type":"object",
+         "required":[
+            "filter"
+         ],
+         "properties":{
+            "filter":{
+               "type":"object",
+               "description":"Filter mapped records before output. ABAC is applied automatically based on fieldMappings.accessFields.",
+               "properties":{
+                  "enforceAbac":{
+                     "type":"boolean",
+                     "default":true
+                  },
+                  "expression":{
+                     "type":"string",
+                     "description":"Optional additional filter expression (e.g. '@.status == \"active\"')."
+                  }
+               },
+               "additionalProperties":false
+            },
+            "lineage":{
+               "$ref":"#/$defs/cipLineage"
+            }
+         },
+         "additionalProperties":false
+      },
+      "cipStepOutput":{
+         "type":"object",
+         "required":[
+            "output"
+         ],
+         "properties":{
+            "output":{
+               "type":"object",
+               "description":"Finalize the CIP pipeline and emit records.",
+               "properties":{
+                  "mode":{
+                     "type":"string",
+                     "enum":[
+                        "records"
+                     ],
+                     "default":"records"
+                  },
+                  "limit":{
+                     "type":"integer",
+                     "minimum":1,
+                     "description":"Optional soft limit for records returned by list operation; sync runner can override."
+                  },
+                  "includeConfidence":{
+                     "type":"boolean",
+                     "default":false,
+                     "description":"If true, the pipeline will compute a confidence score per record based on quality checks, mapping completeness, and source hints."
+                  }
+               },
+               "additionalProperties":false
+            },
+            "lineage":{
+               "$ref":"#/$defs/cipLineage"
+            }
+         },
+         "additionalProperties":false
+      },
+      "cipStepPythonInline":{
+         "type":"object",
+         "required":[
+            "pythonInline"
+         ],
+         "properties":{
+            "pythonInline":{
+               "type":"object",
+               "description":"Sandboxed inline Python transformation, operating on the current record batch.",
+               "properties":{
+                  "code":{
+                     "type":"string",
+                     "description":"Small, pure function snippet; no imports, no IO, no network. Runtime must enforce sandboxing."
+                  }
+               },
+               "required":[
+                  "code"
+               ],
+               "additionalProperties":false
+            },
+            "lineage":{
+               "$ref":"#/$defs/cipLineage"
+            }
+         },
+         "additionalProperties":false
       }
    },
    "additionalProperties":false