@aifabrix/builder 2.39.1 → 2.39.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (18) hide show
  1. package/lib/app/readme.js +50 -8
  2. package/lib/commands/up-common.js +4 -0
  3. package/lib/external-system/generator.js +14 -3
  4. package/lib/schema/external-datasource.schema.json +1593 -1594
  5. package/lib/schema/external-system.schema.json +420 -420
  6. package/package.json +1 -1
  7. package/templates/applications/README.md.hbs +8 -8
  8. package/templates/applications/dataplane/env.template +3 -1
  9. package/templates/applications/dataplane/variables.yaml +1 -1
  10. package/templates/applications/keycloak/variables.yaml +1 -1
  11. package/templates/applications/miso-controller/env.template +14 -0
  12. package/templates/applications/miso-controller/variables.yaml +1 -1
  13. package/templates/external-system/external-system.json.hbs +20 -7
  14. package/templates/applications/dataplane/README.md +0 -219
  15. package/templates/applications/keycloak/README.md +0 -203
  16. package/templates/applications/miso-controller/README.md +0 -367
  17. package/templates/external-system/deploy.ps1.hbs +0 -34
  18. package/templates/external-system/deploy.sh.hbs +0 -34
package/lib/schema/external-datasource.schema.json CHANGED
@@ -1,1594 +1,1593 @@
1
- {
2
- "$schema":"https://json-schema.org/draft/2020-12/schema",
3
- "$id":"https://raw.githubusercontent.com/esystemsdev/aifabrix-builder/refs/heads/main/lib/schema/external-datasource.schema.json",
4
- "title":"External Data Source",
5
- "description":"Configuration for AI Fabrix ExternalDataSource entities. Includes metadata schema, data dimensions, transformation mappings, OpenAPI/MCP exposure, execution logic, and sync behavior.",
6
- "metadata":{
7
- "key":"external-datasource-schema",
8
- "name":"External Data Source Configuration Schema",
9
- "description":"JSON schema for validating ExternalDataSource configuration files",
10
- "version":"2.1.0",
11
- "type":"schema",
12
- "category":"integration",
13
- "author":"AI Fabrix Team",
14
- "createdAt":"2024-01-01T00:00:00Z",
15
- "updatedAt":"2026-01-19T00:00:00Z",
16
- "compatibility":{
17
- "minVersion":"1.0.0",
18
- "maxVersion":"3.0.0",
19
- "deprecated":false
20
- },
21
- "tags":[
22
- "schema",
23
- "external-datasource",
24
- "dataplane",
25
- "integration",
26
- "validation",
27
- "cip",
28
- "execution"
29
- ],
30
- "dependencies":[
31
- "external-system.schema.json"
32
- ],
33
- "changelog":[
34
- {
35
- "version":"1.0.0",
36
- "date":"2024-01-01T00:00:00Z",
37
- "changes":[
38
- "Initial schema for External Data Sources",
39
- "Added fieldMappings DSL with transformation expressions",
40
- "Added metadataSchema validation",
41
- "Added exposed fields configuration for MCP/OpenAPI",
42
- "Added ABAC accessFields validation",
43
- "Added OpenAPI and MCP operation configurations",
44
- "Added sync configuration and validation rules"
45
- ],
46
- "breaking":false
47
- },
48
- {
49
- "version":"1.1.0",
50
- "date":"2025-01-01T00:00:00Z",
51
- "changes":[
52
- "Added execution.engine with CIP and Python execution modes",
53
- "Introduced CIP definition ($defs.cipDefinition and step types)",
54
- "Added capabilities for list/get/create/update/delete",
55
- "Extended fieldMappings with semantic metadata for AI",
56
- "Added exposure profiles for different AI/analytics views",
57
- "Added data quality rules, indexing, and AI context hints"
58
- ],
59
- "breaking":false
60
- },
61
- {
62
- "version":"1.2.0",
63
- "date":"2026-01-07T00:00:00Z",
64
- "changes":[
65
- "Added config.abac section with crossSystemSql and crossSystemJson properties",
66
- "Added support for unified filter model (JSON and SQL formats) in ABAC cross-system configuration",
67
- "Updated CIP filter expression to support both SQL and JSON formats"
68
- ],
69
- "breaking":false
70
- },
71
- {
72
- "version":"2.0.0",
73
- "date":"2026-01-18T00:00:00Z",
74
- "changes":[
75
- "BREAKING: Renamed fieldMappings.accessFields (array) to fieldMappings.dimensions (object)",
76
- "BREAKING: Renamed fieldMappings.fields to fieldMappings.attributes",
77
- "BREAKING: Renamed entityKey to entityType",
78
- "BREAKING: Renamed exposed.fields to exposed.attributes",
79
- "Added indexed property to attribute definitions for database index creation",
80
- "Updated descriptions to reflect dimension-first data model approach"
81
- ],
82
- "breaking":true
83
- },
84
- {
85
- "version":"2.1.0",
86
- "date":"2026-01-19T00:00:00Z",
87
- "changes":[
88
- "Added error semantics (onError object) to all CIP step types with error classification, retry, compensation, and failPipeline",
89
- "Added idempotency configuration to CIP definition for execution-level replay guarantees",
90
- "Added lineage configuration to field mappings for field-level explainability and compliance",
91
- "Added contract versioning configuration to datasource root for CI/CD safety and agent stability"
92
- ],
93
- "breaking":false
94
- }
95
- ]
96
- },
97
- "type":"object",
98
- "required":[
99
- "key",
100
- "displayName",
101
- "systemKey",
102
- "entityType",
103
- "resourceType",
104
- "fieldMappings"
105
- ],
106
- "properties":{
107
- "key":{
108
- "type":"string",
109
- "pattern":"^[a-z0-9-]+$",
110
- "minLength":3,
111
- "description":"Unique key of data source (e.g. 'hubspot-deal')."
112
- },
113
- "displayName":{
114
- "type":"string",
115
- "minLength":1
116
- },
117
- "description":{
118
- "type":"string"
119
- },
120
- "enabled":{
121
- "type":"boolean",
122
- "default":true
123
- },
124
- "systemKey":{
125
- "type":"string",
126
- "pattern":"^[a-z0-9-]+$",
127
- "description":"Must match ExternalSystem.key"
128
- },
129
- "entityType":{
130
- "type":"string",
131
- "enum":["document-storage","documentStorage","vector-store","vectorStore","record-storage","recordStorage","message-service","messageService","none"],
132
- "description":"Entity type identifier. Defines storage semantics and behavior: 'document-storage' or 'documentStorage' (creates document storage service with vector-storage), 'vector-store' or 'vectorStore' (external vector storage system), 'record-storage' or 'recordStorage' (creates record-based system with metadata sync and access rights), 'message-service' or 'messageService' (message service for notifications - Slack, Teams, Email), 'none' (uses external system data directly or connects other data sources). Required field. Supports both camelCase and kebab-case formats for backward compatibility."
133
- },
134
- "resourceType":{
135
- "type":"string",
136
- "pattern":"^[a-z0-9-]+$",
137
- "default":"document",
138
- "description":"Resource type for the datasource. Represents business entity classification. Common values: customer, contact, person, document, deal, record. Any lowercase alphanumeric string with hyphens is allowed."
139
- },
140
- "version":{
141
- "type":"string",
142
- "pattern":"^[0-9]+\\.[0-9]+\\.[0-9]+$",
143
- "default":"1.0.0"
144
- },
145
- "metadataSchema":{
146
- "type":"object",
147
- "description":"Subset of JSON Schema used to validate raw input metadata.",
148
- "additionalProperties":true
149
- },
150
- "fieldMappings":{
151
- "type":"object",
152
- "description":"Transformation rules and data dimensions. Maps canonical dimensions to system attributes.",
153
- "required":[
154
- "dimensions",
155
- "attributes"
156
- ],
157
- "properties":{
158
- "dimensions":{
159
- "type":"object",
160
- "description":"Data dimensions mapping. Key = dimension key (from Dimension Catalog), Value = attribute path (e.g., 'metadata.department').",
161
- "additionalProperties":{
162
- "type":"string",
163
- "pattern":"^[a-zA-Z0-9_.]+$"
164
- },
165
- "minProperties":0
166
- },
167
- "attributes":{
168
- "type":"object",
169
- "description":"Key = normalized attribute name. Value = transformation expression + type.",
170
- "additionalProperties":{
171
- "type":"object",
172
- "required":[
173
- "expression",
174
- "type"
175
- ],
176
- "properties":{
177
- "expression":{
178
- "type":"string",
179
- "description":"Pipe-based DSL expression: '{{raw.path}} | toLower | trim' or record reference: 'record_ref:customer'.",
180
- "pattern":"^\\s*((\\{\\{[^}]+\\}\\}(\\s*\\|\\s*[a-zA-Z0-9_]+(\\([^)]*\\))?)*)|(record_ref:[a-z0-9-]+))\\s*$"
181
- },
182
- "type":{
183
- "type":"string",
184
- "enum":[
185
- "string",
186
- "number",
187
- "integer",
188
- "boolean",
189
- "array",
190
- "object"
191
- ]
192
- },
193
- "indexed":{
194
- "type":"boolean",
195
- "default":false,
196
- "description":"If true, creates database index for fast search/filtering. Dimensions are automatically indexed."
197
- },
198
- "description":{
199
- "type":"string",
200
- "description":"Technical description of the normalized field."
201
- },
202
- "required":{
203
- "type":"boolean"
204
- },
205
- "semantic":{
206
- "type":"object",
207
- "description":"Semantic metadata for AI agents and schema generation.",
208
- "properties":{
209
- "title":{
210
- "type":"string",
211
- "description":"Human-friendly label for this field."
212
- },
213
- "description":{
214
- "type":"string",
215
- "description":"Business-level description of the field and how it is used."
216
- },
217
- "example":{
218
- "description":"Example value for this field.",
219
- "type":["string","number","boolean","object","array","null"]
220
- },
221
- "categories":{
222
- "type":"array",
223
- "description":"Logical categories/tags (e.g. ['sales', 'revenue']).",
224
- "items":{
225
- "type":"string"
226
- }
227
- },
228
- "synonyms":{
229
- "type":"array",
230
- "description":"Optional list of synonyms used in natural language (e.g. 'opportunity', 'sales-case').",
231
- "items":{
232
- "type":"string"
233
- }
234
- }
235
- },
236
- "additionalProperties":false
237
- },
238
- "lineage":{
239
- "$ref":"#/$defs/fieldMappingLineage"
240
- }
241
- }
242
- }
243
- }
244
- }
245
- },
246
- "exposed":{
247
- "type":"object",
248
- "description":"Defines which normalized fields are exposed through MCP/OpenAPI. Ensures ISO27001-safe output and predictable AI model behavior.",
249
- "properties":{
250
- "attributes":{
251
- "type":"array",
252
- "description":"List of normalized attributes (from fieldMappings.attributes keys) that the MCP/OpenAPI layers will return.",
253
- "items":{
254
- "type":"string",
255
- "pattern":"^[a-zA-Z0-9_]+$"
256
- },
257
- "minItems":1,
258
- "uniqueItems":true
259
- },
260
- "omit":{
261
- "type":"array",
262
- "description":"Fields that exist in normalized metadata but must NEVER be exposed by MCP/OpenAPI (e.g., secrets, internal IDs). Overrides 'fields'.",
263
- "items":{
264
- "type":"string",
265
- "pattern":"^[a-zA-Z0-9_]+$"
266
- },
267
- "uniqueItems":true
268
- },
269
- "groups":{
270
- "type":"object",
271
- "description":"Optional logical grouping for OpenAPI/MCP schema generation (e.g., 'default', 'analytics', 'light').",
272
- "additionalProperties":{
273
- "type":"array",
274
- "items":{
275
- "type":"string",
276
- "pattern":"^[a-zA-Z0-9_]+$"
277
- },
278
- "minItems":1,
279
- "uniqueItems":true
280
- }
281
- },
282
- "readonly":{
283
- "type":"array",
284
- "description":"Fields exposed via MCP/OpenAPI but cannot be modified via write operations.",
285
- "items":{
286
- "type":"string",
287
- "pattern":"^[a-zA-Z0-9_]+$"
288
- },
289
- "uniqueItems":true
290
- },
291
- "description":{
292
- "type":"string",
293
- "description":"Human-readable description of the exposure model for this data source."
294
- },
295
- "profiles":{
296
- "type":"object",
297
- "description":"Named exposure profiles mapping profile keys to lists of normalized field names. Useful for AI-light, AI-embedding, analytics, etc.",
298
- "additionalProperties":{
299
- "type":"array",
300
- "items":{
301
- "type":"string",
302
- "pattern":"^[a-zA-Z0-9_]+$"
303
- },
304
- "minItems":1,
305
- "uniqueItems":true
306
- }
307
- }
308
- },
309
- "required":[
310
- "attributes"
311
- ],
312
- "additionalProperties":false
313
- },
314
- "sync":{
315
- "type":"object",
316
- "description":"Record synchronization rules.",
317
- "properties":{
318
- "mode":{
319
- "type":"string",
320
- "enum":[
321
- "pull",
322
- "push",
323
- "bidirectional"
324
- ],
325
- "default":"pull"
326
- },
327
- "schedule":{
328
- "type":"string"
329
- },
330
- "batchSize":{
331
- "type":"integer",
332
- "minimum":1,
333
- "maximum":10000,
334
- "default":500
335
- },
336
- "maxParallelRequests":{
337
- "type":"integer",
338
- "minimum":1,
339
- "maximum":50,
340
- "default":5
341
- }
342
- }
343
- },
344
- "openapi":{
345
- "type":"object",
346
- "description":"OpenAPI-driven connector configuration. Only includes endpoints selected during wizard onboarding. To add more endpoints, run the wizard again.",
347
- "properties":{
348
- "enabled":{
349
- "type":"boolean",
350
- "default":false
351
- },
352
- "documentKey":{
353
- "type":"string"
354
- },
355
- "baseUrl":{
356
- "type":"string",
357
- "pattern":"^(http|https)://"
358
- },
359
- "resourcePath":{
360
- "type":"string"
361
- },
362
- "operations":{
363
- "type":"object",
364
- "description":"Selected operations from the OpenAPI document. Only endpoints chosen during wizard onboarding are included.",
365
- "properties":{
366
- "list":{
367
- "type":"object",
368
- "required":[
369
- "operationId"
370
- ],
371
- "properties":{
372
- "operationId":{
373
- "type":"string"
374
- },
375
- "method":{
376
- "type":"string",
377
- "enum":[
378
- "GET",
379
- "POST"
380
- ]
381
- },
382
- "path":{
383
- "type":"string"
384
- }
385
- }
386
- },
387
- "get":{
388
- "type":"object",
389
- "required":[
390
- "operationId"
391
- ],
392
- "properties":{
393
- "operationId":{
394
- "type":"string"
395
- },
396
- "method":{
397
- "type":"string",
398
- "enum":[
399
- "GET"
400
- ]
401
- },
402
- "path":{
403
- "type":"string"
404
- }
405
- }
406
- },
407
- "create":{
408
- "type":"object",
409
- "properties":{
410
- "operationId":{
411
- "type":"string"
412
- },
413
- "method":{
414
- "type":"string",
415
- "enum":[
416
- "POST",
417
- "PUT"
418
- ]
419
- },
420
- "path":{
421
- "type":"string"
422
- }
423
- }
424
- },
425
- "update":{
426
- "type":"object",
427
- "properties":{
428
- "operationId":{
429
- "type":"string"
430
- },
431
- "method":{
432
- "type":"string",
433
- "enum":[
434
- "PATCH",
435
- "PUT",
436
- "POST"
437
- ]
438
- },
439
- "path":{
440
- "type":"string"
441
- }
442
- }
443
- },
444
- "delete":{
445
- "type":"object",
446
- "properties":{
447
- "operationId":{
448
- "type":"string"
449
- },
450
- "method":{
451
- "type":"string",
452
- "enum":[
453
- "DELETE"
454
- ]
455
- },
456
- "path":{
457
- "type":"string"
458
- }
459
- }
460
- }
461
- }
462
- },
463
- "autoRbac":{
464
- "type":"boolean",
465
- "default":false,
466
- "description":"Generates <system>.<entity>.<action> RBAC permissions."
467
- }
468
- }
469
- },
470
- "validation":{
471
- "type":"object",
472
- "description":"Advanced validation rules applied before saving normalized metadata.",
473
- "properties":{
474
- "repeatingValues":{
475
- "type":"array",
476
- "items":{
477
- "type":"object",
478
- "required":[
479
- "field",
480
- "scope",
481
- "strategy"
482
- ],
483
- "properties":{
484
- "field":{
485
- "type":"string",
486
- "pattern":"^[a-zA-Z0-9_]+$"
487
- },
488
- "scope":{
489
- "type":"array",
490
- "items":{
491
- "type":"string"
492
- }
493
- },
494
- "strategy":{
495
- "type":"string",
496
- "enum":[
497
- "reject",
498
- "latest-wins",
499
- "first-wins",
500
- "merge"
501
- ]
502
- },
503
- "mergeFields":{
504
- "type":"array",
505
- "items":{
506
- "type":"string"
507
- }
508
- }
509
- }
510
- }
511
- }
512
- }
513
- },
514
- "quality":{
515
- "type":"object",
516
- "description":"Data quality rules applied after mapping but before persistence and AI exposure.",
517
- "properties":{
518
- "rejectIf":{
519
- "type":"array",
520
- "description":"List of conditions that cause a record to be rejected.",
521
- "items":{
522
- "type":"object",
523
- "required":[
524
- "field",
525
- "operator"
526
- ],
527
- "properties":{
528
- "field":{
529
- "type":"string",
530
- "pattern":"^[a-zA-Z0-9_]+$",
531
- "description":"Normalized field name to evaluate."
532
- },
533
- "operator":{
534
- "type":"string",
535
- "enum":[
536
- "missing",
537
- "null",
538
- "empty",
539
- "invalidRef",
540
- "lessThan",
541
- "greaterThan",
542
- "equal",
543
- "notEqual"
544
- ]
545
- },
546
- "value":{
547
- "description":"Comparison value for applicable operators.",
548
- "type":["string","number","boolean","null"]
549
- },
550
- "message":{
551
- "type":"string",
552
- "description":"Optional human-readable reason for rejection."
553
- }
554
- },
555
- "additionalProperties":false
556
- }
557
- }
558
- },
559
- "additionalProperties":false
560
- },
561
- "indexing":{
562
- "type":"object",
563
- "description":"Indexing and embedding strategy for this datasource.",
564
- "properties":{
565
- "embedding":{
566
- "type":"array",
567
- "description":"List of normalized fields whose values should be concatenated and embedded for vector search.",
568
- "items":{
569
- "type":"string",
570
- "pattern":"^[a-zA-Z0-9_]+$"
571
- },
572
- "minItems":1,
573
- "uniqueItems":true
574
- },
575
- "uniqueKey":{
576
- "type":"string",
577
- "pattern":"^[a-zA-Z0-9_]+$",
578
- "description":"Normalized field used as unique identifier for deduplication and upserts."
579
- },
580
- "dedupeStrategy":{
581
- "type":"string",
582
- "enum":[
583
- "reject",
584
- "latest-wins",
585
- "first-wins",
586
- "merge"
587
- ],
588
- "default":"latest-wins",
589
- "description":"Strategy to apply when multiple records share the same uniqueKey."
590
- }
591
- },
592
- "additionalProperties":false
593
- },
594
- "context":{
595
- "type":"object",
596
- "description":"Natural-language and semantic hints for AI agents.",
597
- "properties":{
598
- "semanticTags":{
599
- "type":"array",
600
- "description":"High-level tags describing this datasource (e.g. ['crm', 'customer-data', 'deal']).",
601
- "items":{
602
- "type":"string"
603
- }
604
- },
605
- "synonyms":{
606
- "type":"array",
607
- "description":"Terms users might use in natural language to refer to this datasource or its primary entity.",
608
- "items":{
609
- "type":"string"
610
- }
611
- },
612
- "naturalLanguageHints":{
613
- "type":"array",
614
- "description":"Free-form hints for LLMs on how to interpret or use this datasource.",
615
- "items":{
616
- "type":"string"
617
- }
618
- }
619
- },
620
- "additionalProperties":false
621
- },
622
- "documentStorage":{
623
- "type":"object",
624
- "description":"Document storage configuration (optional, enables vector storage). Validated against type/document-storage.json schema.",
625
- "properties":{
626
- "enabled":{
627
- "type":"boolean",
628
- "default":true
629
- },
630
- "twoPhaseSync":{
631
- "type":"boolean",
632
- "default":true,
633
- "description":"Enable two-phase sync pattern. When true: validates metadata first (quality rules, comparison with DocumentRecords), then fetches binaries via CIP for changed/new documents. When false: fetches binaries directly without metadata validation phase (single-phase sync). Note: Files are never synced back to external systems (one-way sync only: external → dataplane)."
634
- },
635
- "binaryOperationRef":{
636
- "type":"string",
637
- "default":"get",
638
- "description":"CIP operation name for binary document retrieval. Must exist in execution.cip.operations. Defaults to 'get' operation."
639
- },
640
- "responseType":{
641
- "type":"string",
642
- "enum":[
643
- "binary",
644
- "base64",
645
- "json"
646
- ],
647
- "default":"binary",
648
- "description":"Expected response type from CIP operation. 'binary' for raw binary data, 'base64' for base64-encoded data, 'json' for JSON response with binary field."
649
- },
650
- "binaryField":{
651
- "type":"string",
652
- "description":"Field name containing binary data if responseType is 'json' or 'base64'. Required when responseType is not 'binary'."
653
- },
654
- "parameterMapping":{
655
- "type":"object",
656
- "additionalProperties":{
657
- "type":"string"
658
- },
659
- "description":"Map metadata record fields to CIP operation parameters. Example: {\"fileId\": \"{{key}}\", \"downloadUrl\": \"{{metadata.downloadUrl}}\"}"
660
- },
661
- "processing":{
662
- "type":"object",
663
- "properties":{
664
- "fileStoragePath":{
665
- "type":"string",
666
- "default":"/data/documents"
667
- },
668
- "aiValidation":{
669
- "type":"object"
670
- },
671
- "spacyEnrichment":{
672
- "type":"object"
673
- },
674
- "notifications":{
675
- "type":"object"
676
- }
677
- },
678
- "additionalProperties":false
679
- },
680
- "credentialId":{
681
- "type":"string"
682
- }
683
- },
684
- "required":[
685
- "enabled"
686
- ],
687
- "additionalProperties":false
688
- },
689
- "portalInput":{
690
- "type":"array",
691
- "description":"Optional UI metadata definition for the AI Fabrix portal.",
692
- "items":{
693
- "type":"object",
694
- "required":[
695
- "name",
696
- "field",
697
- "label"
698
- ],
699
- "properties":{
700
- "name":{
701
- "type":"string"
702
- },
703
- "field":{
704
- "type":"string",
705
- "enum":[
706
- "text",
707
- "textarea",
708
- "select",
709
- "json",
710
- "boolean",
711
- "number"
712
- ]
713
- },
714
- "label":{
715
- "type":"string"
716
- },
717
- "placeholder":{
718
- "type":"string"
719
- },
720
- "options":{
721
- "type":"array",
722
- "items":{
723
- "type":"string"
724
- }
725
- },
726
- "masked":{
727
- "type":"boolean"
728
- },
729
- "validation":{
730
- "type":"object",
731
- "properties":{
732
- "minLength":{
733
- "type":"integer"
734
- },
735
- "maxLength":{
736
- "type":"integer"
737
- },
738
- "pattern":{
739
- "type":"string"
740
- },
741
- "required":{
742
- "type":"boolean"
743
- }
744
- }
745
- }
746
- }
747
- }
748
- },
749
- "testPayload":{
750
- "type":"object",
751
- "description":"Test payload configuration for unit and integration testing",
752
- "properties":{
753
- "payloadTemplate":{
754
- "type":"object",
755
- "description":"Sample payload matching the expected API response structure. Used for testing field mappings and metadata schema validation.",
756
- "additionalProperties":true
757
- },
758
- "expectedResult":{
759
- "type":"object",
760
- "description":"Expected normalized result after field mapping transformations (optional, for validation)",
761
- "additionalProperties":true
762
- }
763
- },
764
- "additionalProperties":false
765
- },
766
- "capabilities":{
767
- "type":"object",
768
- "description":"Declares which logical operations are supported by this datasource.",
769
- "properties":{
770
- "list":{
771
- "type":"boolean",
772
- "default":true
773
- },
774
- "get":{
775
- "type":"boolean",
776
- "default":false
777
- },
778
- "create":{
779
- "type":"boolean",
780
- "default":false
781
- },
782
- "update":{
783
- "type":"boolean",
784
- "default":false
785
- },
786
- "delete":{
787
- "type":"boolean",
788
- "default":false
789
- }
790
- },
791
- "additionalProperties":false
792
- },
793
- "execution":{
794
- "type":"object",
795
- "description":"Execution engine configuration for this datasource (CIP or Python). CIP is the native declarative mode.",
796
- "required":[
797
- "engine"
798
- ],
799
- "properties":{
800
- "engine":{
801
- "type":"string",
802
- "enum":[
803
- "cip",
804
- "python"
805
- ],
806
- "description":"Execution engine. 'cip' for declarative pipelines, 'python' for custom handlers."
807
- },
808
- "cip":{
809
- "$ref":"#/$defs/cipDefinition"
810
- },
811
- "python":{
812
- "$ref":"#/$defs/pythonExecution"
813
- }
814
- },
815
- "additionalProperties":false
816
- },
817
- "config":{
818
- "type":"object",
819
- "description":"Additional configuration for this datasource, including ABAC settings, MCP contracts, and other metadata.",
820
- "properties":{
821
- "abac":{
822
- "type":"object",
823
- "description":"Attribute-Based Access Control (ABAC) configuration for this datasource.",
824
- "properties":{
825
- "dimensions":{
826
- "type":"object",
827
- "description":"Data dimensions mapping. Key = dimension key (from Dimension Catalog), Value = attribute path. Overrides fieldMappings.dimensions if specified.",
828
- "additionalProperties":{
829
- "type":"string",
830
- "pattern":"^[a-zA-Z0-9_.]+$"
831
- }
832
- },
833
- "crossSystemSql":{
834
- "type":"string",
835
- "description":"Cross-system ABAC filter expression in SQL format (advanced, for developers). Example: 'hubspot-companies.country = user.country AND hubspot-companies.revenue >= 1000000'"
836
- },
837
- "crossSystemJson":{
838
- "type":"object",
839
- "description":"Cross-system ABAC filter expression in JSON format (simple, for UI). Example: {'hubspot-companies.country': {'eq': 'user.country'}, 'hubspot-companies.revenue': {'gte': 1000000}}",
840
- "additionalProperties":{
841
- "type":"object",
842
- "properties":{
843
- "eq":{"type":["string","number","boolean"]},
844
- "ne":{"type":["string","number","boolean"]},
845
- "gt":{"type":["string","number"]},
846
- "lt":{"type":["string","number"]},
847
- "gte":{"type":["string","number"]},
848
- "lte":{"type":["string","number"]},
849
- "in":{"type":"array"},
850
- "nin":{"type":"array"},
851
- "contains":{"type":"string"},
852
- "like":{"type":"string"},
853
- "isNull":{"type":"null"},
854
- "isNotNull":{"type":"null"}
855
- }
856
- }
857
- }
858
- },
859
- "additionalProperties":false
860
- }
861
- },
862
- "additionalProperties":true
863
- },
864
- "contract":{
865
- "$ref":"#/$defs/contractConfig"
866
- }
867
- },
868
- "$defs":{
869
- "pythonExecution":{
870
- "type":"object",
871
- "description":"Python-based execution config for advanced/custom use cases.",
872
- "properties":{
873
- "entrypoint":{
874
- "type":"string",
875
- "description":"Python callable in module notation, e.g. 'handlers.hubspot_deal.list_records'."
876
- },
877
- "codeRef":{
878
- "type":"string",
879
- "description":"Optional reference to external code bundle identifier or embedded block key."
880
- }
881
- },
882
- "required":[
883
- "entrypoint"
884
- ],
885
- "additionalProperties":false
886
- },
887
- "cipLineage":{
888
- "type":"object",
889
- "description":"Optional lineage metadata for a CIP operation or step.",
890
- "properties":{
891
- "producer":{
892
- "type":"string",
893
- "description":"Upstream system or connector name (e.g. 'hubspot')."
894
- },
895
- "entity":{
896
- "type":"string",
897
- "description":"Upstream entity or resource (e.g. 'deal')."
898
- },
899
- "operation":{
900
- "type":"string",
901
- "description":"Logical operation (e.g. 'list', 'get', 'sync-job-1')."
902
- }
903
- },
904
- "additionalProperties":false
905
- },
906
- "cipDefinition":{
907
- "type":"object",
908
- "description":"Composable Integration Pipeline (CIP) definition for this datasource.",
909
- "required":[
910
- "version",
911
- "operations"
912
- ],
913
- "properties":{
914
- "version":{
915
- "type":"string",
916
- "enum":[
917
- "1.0"
918
- ]
919
- },
920
- "inputType":{
921
- "type":"string",
922
- "const":"void",
923
- "description":"Logical input type of the pipeline. Reserved for future extensions."
924
- },
925
- "outputType":{
926
- "type":"string",
927
- "const":"records",
928
- "description":"Logical output type of the pipeline."
929
- },
930
- "operations":{
931
- "type":"object",
932
- "description":"Logical operations (list/get/create/update/delete) implemented for this datasource.",
933
- "properties":{
934
- "list":{
935
- "$ref":"#/$defs/cipOperation"
936
- },
937
- "get":{
938
- "$ref":"#/$defs/cipOperation"
939
- },
940
- "create":{
941
- "$ref":"#/$defs/cipOperation"
942
- },
943
- "update":{
944
- "$ref":"#/$defs/cipOperation"
945
- },
946
- "delete":{
947
- "$ref":"#/$defs/cipOperation"
948
- }
949
- },
950
- "additionalProperties":false
951
- },
952
- "idempotency":{
953
- "$ref":"#/$defs/idempotencyConfig"
954
- }
955
- },
956
- "additionalProperties":false
957
- },
958
- "cipOperation":{
959
- "type":"object",
960
- "required":[
961
- "steps"
962
- ],
963
- "properties":{
964
- "description":{
965
- "type":"string"
966
- },
967
- "enabled":{
968
- "type":"boolean",
969
- "default":true
970
- },
971
- "lineage":{
972
- "$ref":"#/$defs/cipLineage"
973
- },
974
- "steps":{
975
- "type":"array",
976
- "minItems":1,
977
- "items":{
978
- "$ref":"#/$defs/cipStep"
979
- }
980
- }
981
- },
982
- "additionalProperties":false
983
- },
984
- "cipStep":{
985
- "description":"One step in the CIP pipeline.",
986
- "oneOf":[
987
- {
988
- "$ref":"#/$defs/cipStepFetch"
989
- },
990
- {
991
- "$ref":"#/$defs/cipStepPaginate"
992
- },
993
- {
994
- "$ref":"#/$defs/cipStepMap"
995
- },
996
- {
997
- "$ref":"#/$defs/cipStepFilter"
998
- },
999
- {
1000
- "$ref":"#/$defs/cipStepOutput"
1001
- },
1002
- {
1003
- "$ref":"#/$defs/cipStepPythonInline"
1004
- }
1005
- ]
1006
- },
1007
- "cipStepFetch":{
1008
- "type":"object",
1009
- "required":[
1010
- "fetch"
1011
- ],
1012
- "properties":{
1013
- "fetch":{
1014
- "type":"object",
1015
- "description":"Fetch raw records from an upstream system.",
1016
- "required":[
1017
- "source"
1018
- ],
1019
- "properties":{
1020
- "source":{
1021
- "type":"string",
1022
- "enum":[
1023
- "openapi",
1024
- "http"
1025
- ],
1026
- "description":"If 'openapi', refer to openapi.operations.*. If 'http', use explicit method/path."
1027
- },
1028
- "openapiRef":{
1029
- "type":"string",
1030
- "enum":[
1031
- "list",
1032
- "get",
1033
- "create",
1034
- "update",
1035
- "delete"
1036
- ],
1037
- "description":"Reference to openapi.operations.<openapiRef> entry."
1038
- },
1039
- "operationId":{
1040
- "type":"string",
1041
- "description":"OpenAPI operationId to call when source='openapi'. Optional when openapiRef is used."
1042
- },
1043
- "method":{
1044
- "type":"string",
1045
- "enum":[
1046
- "GET",
1047
- "POST",
1048
- "PUT",
1049
- "PATCH",
1050
- "DELETE"
1051
- ],
1052
- "description":"Explicit HTTP method when source='http'."
1053
- },
1054
- "path":{
1055
- "type":"string",
1056
- "description":"Explicit HTTP path when source='http'."
1057
- },
1058
- "query":{
1059
- "type":"object",
1060
- "description":"Static query parameters. ABAC/runtime can still override or append at runtime.",
1061
- "additionalProperties":{
1062
- "type":[
1063
- "string",
1064
- "number",
1065
- "boolean",
1066
- "null"
1067
- ]
1068
- }
1069
- },
1070
- "bodyTemplate":{
1071
- "description":"Optional static request body or template for POST/PUT/PATCH fetches.",
1072
- "type":["object","string","null"]
1073
- },
1074
- "headers":{
1075
- "type":"object",
1076
- "description":"Optional HTTP headers for the request.",
1077
- "additionalProperties":{
1078
- "type":"string"
1079
- }
1080
- }
1081
- },
1082
- "allOf":[
1083
- {
1084
- "if":{
1085
- "properties":{
1086
- "source":{
1087
- "const":"openapi"
1088
- }
1089
- }
1090
- },
1091
- "then":{
1092
- "anyOf":[
1093
- {
1094
- "required":[
1095
- "openapiRef"
1096
- ]
1097
- },
1098
- {
1099
- "required":[
1100
- "operationId"
1101
- ]
1102
- }
1103
- ]
1104
- }
1105
- },
1106
- {
1107
- "if":{
1108
- "properties":{
1109
- "source":{
1110
- "const":"http"
1111
- }
1112
- }
1113
- },
1114
- "then":{
1115
- "required":[
1116
- "method",
1117
- "path"
1118
- ]
1119
- }
1120
- }
1121
- ],
1122
- "additionalProperties":false
1123
- },
1124
- "lineage":{
1125
- "$ref":"#/$defs/cipLineage"
1126
- },
1127
- "stepId":{
1128
- "type":"string",
1129
- "pattern":"^[a-z0-9-_]+$",
1130
- "description":"Unique identifier for this step within the operation"
1131
- },
1132
- "onError":{
1133
- "$ref":"#/$defs/cipOnErrorConfig"
1134
- }
1135
- },
1136
- "additionalProperties":false
1137
- },
1138
- "cipStepPaginate":{
1139
- "type":"object",
1140
- "required":[
1141
- "paginate"
1142
- ],
1143
- "properties":{
1144
- "paginate":{
1145
- "type":"object",
1146
- "description":"Pagination strategy applied to results from the preceding fetch.",
1147
- "required":[
1148
- "strategy"
1149
- ],
1150
- "properties":{
1151
- "strategy":{
1152
- "type":"string",
1153
- "enum":[
1154
- "cursor",
1155
- "page",
1156
- "offset",
1157
- "none"
1158
- ]
1159
- },
1160
- "cursorField":{
1161
- "type":"string",
1162
- "description":"Field in the response payload containing the cursor value (for 'cursor' strategy)."
1163
- },
1164
- "cursorParam":{
1165
- "type":"string",
1166
- "description":"Name of the query parameter used to send the cursor."
1167
- },
1168
- "pageParam":{
1169
- "type":"string",
1170
- "description":"Name of the query parameter used to send the page number (for 'page' strategy)."
1171
- },
1172
- "pageSizeParam":{
1173
- "type":"string",
1174
- "description":"Name of the query parameter used to send the page size."
1175
- },
1176
- "pageSize":{
1177
- "type":"integer",
1178
- "minimum":1,
1179
- "description":"Default page size."
1180
- },
1181
- "maxPages":{
1182
- "type":"integer",
1183
- "minimum":1,
1184
- "description":"Maximum number of pages to fetch."
1185
- }
1186
- },
1187
- "additionalProperties":false
1188
- },
1189
- "lineage":{
1190
- "$ref":"#/$defs/cipLineage"
1191
- },
1192
- "stepId":{
1193
- "type":"string",
1194
- "pattern":"^[a-z0-9-_]+$",
1195
- "description":"Unique identifier for this step within the operation"
1196
- },
1197
- "onError":{
1198
- "$ref":"#/$defs/cipOnErrorConfig"
1199
- }
1200
- },
1201
- "additionalProperties":false
1202
- },
1203
- "cipStepMap":{
1204
- "type":"object",
1205
- "required":[
1206
- "map"
1207
- ],
1208
- "properties":{
1209
- "map":{
1210
- "type":"object",
1211
- "description":"Transform raw API records into the normalized metadata model.",
1212
- "properties":{
1213
- "inputPath":{
1214
- "type":"string",
1215
- "default":"$.items[*]",
1216
- "description":"JSONPath-like expression to iterate over records in the upstream payload."
1217
- },
1218
- "useFieldMappings":{
1219
- "type":"boolean",
1220
- "default":true,
1221
- "description":"If true, apply the datasource.fieldMappings.attributes expressions; no inline mapping needed."
1222
- },
1223
- "inline":{
1224
- "type":"object",
1225
- "description":"Optional inline mapping overrides { normalizedField -> expression }.",
1226
- "additionalProperties":{
1227
- "type":"string",
1228
- "description":"Pipe-based DSL expression overriding fieldMappings expression."
1229
- }
1230
- }
1231
- },
1232
- "additionalProperties":false
1233
- },
1234
- "lineage":{
1235
- "$ref":"#/$defs/cipLineage"
1236
- },
1237
- "stepId":{
1238
- "type":"string",
1239
- "pattern":"^[a-z0-9-_]+$",
1240
- "description":"Unique identifier for this step within the operation"
1241
- },
1242
- "onError":{
1243
- "$ref":"#/$defs/cipOnErrorConfig"
1244
- }
1245
- },
1246
- "additionalProperties":false
1247
- },
1248
- "cipStepFilter":{
1249
- "type":"object",
1250
- "required":[
1251
- "filter"
1252
- ],
1253
- "properties":{
1254
- "filter":{
1255
- "type":"object",
1256
- "description":"Filter mapped records before output. Dimension enforcement is applied automatically based on fieldMappings.dimensions.",
1257
- "properties":{
1258
- "enforceAbac":{
1259
- "type":"boolean",
1260
- "default":true
1261
- },
1262
- "expression":{
1263
- "oneOf":[
1264
- {
1265
- "type":"string",
1266
- "description":"SQL filter expression (e.g., 'status = \"active\" AND revenue >= 1000000')"
1267
- },
1268
- {
1269
- "type":"object",
1270
- "description":"JSON filter expression (e.g., {'status': {'eq': 'active'}, 'revenue': {'gte': 1000000}})",
1271
- "additionalProperties":{
1272
- "type":"object"
1273
- }
1274
- }
1275
- ],
1276
- "description":"Optional additional filter expression. Supports both SQL and JSON formats."
1277
- }
1278
- },
1279
- "additionalProperties":false
1280
- },
1281
- "lineage":{
1282
- "$ref":"#/$defs/cipLineage"
1283
- },
1284
- "stepId":{
1285
- "type":"string",
1286
- "pattern":"^[a-z0-9-_]+$",
1287
- "description":"Unique identifier for this step within the operation"
1288
- },
1289
- "onError":{
1290
- "$ref":"#/$defs/cipOnErrorConfig"
1291
- }
1292
- },
1293
- "additionalProperties":false
1294
- },
1295
- "cipStepOutput":{
1296
- "type":"object",
1297
- "required":[
1298
- "output"
1299
- ],
1300
- "properties":{
1301
- "output":{
1302
- "type":"object",
1303
- "description":"Finalize the CIP pipeline and emit records.",
1304
- "properties":{
1305
- "mode":{
1306
- "type":"string",
1307
- "enum":[
1308
- "records"
1309
- ],
1310
- "default":"records"
1311
- },
1312
- "limit":{
1313
- "type":"integer",
1314
- "minimum":1,
1315
- "description":"Optional soft limit for records returned by list operation; sync runner can override."
1316
- },
1317
- "includeConfidence":{
1318
- "type":"boolean",
1319
- "default":false,
1320
- "description":"If true, the pipeline will compute a confidence score per record based on quality checks, mapping completeness, and source hints."
1321
- }
1322
- },
1323
- "additionalProperties":false
1324
- },
1325
- "lineage":{
1326
- "$ref":"#/$defs/cipLineage"
1327
- },
1328
- "stepId":{
1329
- "type":"string",
1330
- "pattern":"^[a-z0-9-_]+$",
1331
- "description":"Unique identifier for this step within the operation"
1332
- },
1333
- "onError":{
1334
- "$ref":"#/$defs/cipOnErrorConfig"
1335
- }
1336
- },
1337
- "additionalProperties":false
1338
- },
1339
- "cipStepPythonInline":{
1340
- "type":"object",
1341
- "required":[
1342
- "pythonInline"
1343
- ],
1344
- "properties":{
1345
- "pythonInline":{
1346
- "type":"object",
1347
- "description":"Sandboxed inline Python transformation, operating on the current record batch.",
1348
- "properties":{
1349
- "code":{
1350
- "type":"string",
1351
- "description":"Small, pure function snippet; no imports, no IO, no network. Runtime must enforce sandboxing."
1352
- }
1353
- },
1354
- "required":[
1355
- "code"
1356
- ],
1357
- "additionalProperties":false
1358
- },
1359
- "lineage":{
1360
- "$ref":"#/$defs/cipLineage"
1361
- },
1362
- "stepId":{
1363
- "type":"string",
1364
- "pattern":"^[a-z0-9-_]+$",
1365
- "description":"Unique identifier for this step within the operation"
1366
- },
1367
- "onError":{
1368
- "$ref":"#/$defs/cipOnErrorConfig"
1369
- }
1370
- },
1371
- "additionalProperties":false
1372
- },
1373
- "cipRetryConfig":{
1374
- "type":"object",
1375
- "description":"Retry configuration for error handling.",
1376
- "properties":{
1377
- "attempts":{
1378
- "type":"integer",
1379
- "minimum":1,
1380
- "maximum":10,
1381
- "default":3,
1382
- "description":"Maximum number of retry attempts"
1383
- },
1384
- "backoff":{
1385
- "type":"string",
1386
- "enum":["exponential","linear","fixed"],
1387
- "default":"exponential",
1388
- "description":"Backoff strategy: exponential (2^attempt), linear (attempt * base), fixed (constant)"
1389
- },
1390
- "maxDelaySeconds":{
1391
- "type":"number",
1392
- "minimum":1.0,
1393
- "maximum":300.0,
1394
- "default":60.0,
1395
- "description":"Maximum delay between retries in seconds"
1396
- }
1397
- },
1398
- "additionalProperties":false
1399
- },
1400
- "cipCompensationConfig":{
1401
- "type":"object",
1402
- "description":"Compensation (rollback) configuration for error handling.",
1403
- "properties":{
1404
- "enabled":{
1405
- "type":"boolean",
1406
- "default":false,
1407
- "description":"Whether compensation is enabled"
1408
- },
1409
- "stepRef":{
1410
- "type":"string",
1411
- "pattern":"^[a-z0-9-_]+$",
1412
- "description":"Reference to a previously executed step by stepId to execute as compensation"
1413
- }
1414
- },
1415
- "additionalProperties":false
1416
- },
1417
- "cipOnErrorConfig":{
1418
- "type":"object",
1419
- "description":"Enhanced error handling configuration for CIP steps.",
1420
- "required":["class"],
1421
- "properties":{
1422
- "class":{
1423
- "type":"string",
1424
- "enum":["transient","permanent","validation","auth"],
1425
- "description":"Error classification: transient (retryable), permanent (non-retryable), validation (data validation failure), auth (authentication/authorization failure)"
1426
- },
1427
- "retry":{
1428
- "$ref":"#/$defs/cipRetryConfig",
1429
- "description":"Retry configuration (only applies when class='transient')"
1430
- },
1431
- "compensate":{
1432
- "$ref":"#/$defs/cipCompensationConfig",
1433
- "description":"Compensation (rollback) configuration"
1434
- },
1435
- "failPipeline":{
1436
- "type":"boolean",
1437
- "default":true,
1438
- "description":"Whether step failure should fail the entire pipeline (false allows partial success)"
1439
- }
1440
- },
1441
- "additionalProperties":false
1442
- },
1443
- "idempotencyConfig":{
1444
- "type":"object",
1445
- "description":"Idempotency configuration for CIP execution.",
1446
- "required":["key"],
1447
- "properties":{
1448
- "key":{
1449
- "type":"string",
1450
- "description":"Idempotency key template. Supports variables: {{actor.userId}}, {{raw.requestId}}, {{correlationId}}, {{execution.operation}}, {{execution.datasourceId}}"
1451
- },
1452
- "scope":{
1453
- "type":"string",
1454
- "enum":["datasource","system"],
1455
- "default":"datasource",
1456
- "description":"Idempotency scope: datasource (same key within datasource) or system (same key across all datasources)"
1457
- },
1458
- "ttlSeconds":{
1459
- "type":"integer",
1460
- "minimum":1,
1461
- "maximum":604800,
1462
- "default":86400,
1463
- "description":"Time to live for idempotency records in seconds (default: 86400 = 24 hours)"
1464
- },
1465
- "enforcement":{
1466
- "type":"string",
1467
- "enum":["strict","best-effort"],
1468
- "default":"strict",
1469
- "description":"Enforcement mode: strict (return previous result, no execution) or best-effort (warn + continue)"
1470
- }
1471
- },
1472
- "additionalProperties":false
1473
- },
1474
- "transformationRule":{
1475
- "type":"object",
1476
- "description":"Transformation rule applied to a field.",
1477
- "required":["type"],
1478
- "properties":{
1479
- "type":{
1480
- "type":"string",
1481
- "description":"Type of transformation: 'mapping', 'function', 'filter', 'aggregate'"
1482
- },
1483
- "function":{
1484
- "type":"string",
1485
- "description":"Function name (e.g., 'mapStatus', 'toLower', 'trim')"
1486
- },
1487
- "rule":{
1488
- "type":"string",
1489
- "description":"Human-readable rule description (e.g., 'open → OPEN, closed → CLOSED')"
1490
- }
1491
- },
1492
- "additionalProperties":false
1493
- },
1494
- "fieldMappingLineage":{
1495
- "type":"object",
1496
- "description":"Lineage metadata for a field mapping.",
1497
- "required":["sourceFields"],
1498
- "properties":{
1499
- "sourceFields":{
1500
- "type":"array",
1501
- "minItems":1,
1502
- "items":{
1503
- "type":"string"
1504
- },
1505
- "description":"Source field paths that contribute to this mapped field (e.g., ['raw.fields.state'])"
1506
- },
1507
- "transformations":{
1508
- "type":"array",
1509
- "items":{
1510
- "$ref":"#/$defs/transformationRule"
1511
- },
1512
- "description":"List of transformations applied to source fields"
1513
- },
1514
- "confidence":{
1515
- "type":"number",
1516
- "minimum":0.0,
1517
- "maximum":1.0,
1518
- "default":1.0,
1519
- "description":"Confidence score for this mapping (0.0-1.0). Default: 1.0 for deterministic transformations."
1520
- }
1521
- },
1522
- "additionalProperties":false
1523
- },
1524
- "compatibilityConfig":{
1525
- "type":"object",
1526
- "description":"Compatibility configuration for contract versioning.",
1527
- "properties":{
1528
- "backwardCompatible":{
1529
- "type":"boolean",
1530
- "default":true,
1531
- "description":"Whether this version is backward compatible with previous versions"
1532
- },
1533
- "breakingChanges":{
1534
- "type":"array",
1535
- "items":{
1536
- "type":"string"
1537
- },
1538
- "description":"List of breaking change types: removeField, changeType, changeSemantics"
1539
- }
1540
- },
1541
- "additionalProperties":false
1542
- },
1543
- "deprecationConfig":{
1544
- "type":"object",
1545
- "description":"Deprecation configuration for contract versioning.",
1546
- "properties":{
1547
- "sunsetDate":{
1548
- "type":"string",
1549
- "pattern":"^\\d{4}-\\d{2}-\\d{2}$",
1550
- "description":"Date when this contract version will be sunset (YYYY-MM-DD)"
1551
- },
1552
- "replacedBy":{
1553
- "type":"string",
1554
- "description":"Contract name/version that replaces this deprecated version"
1555
- }
1556
- },
1557
- "additionalProperties":false
1558
- },
1559
- "contractConfig":{
1560
- "type":"object",
1561
- "description":"Contract versioning configuration for datasource.",
1562
- "required":["name","version"],
1563
- "properties":{
1564
- "name":{
1565
- "type":"string",
1566
- "pattern":"^[a-z0-9.-]+$",
1567
- "description":"Contract identifier (e.g., 'jira.issue', separate from datasource key)"
1568
- },
1569
- "version":{
1570
- "type":"string",
1571
- "pattern":"^[0-9]+\\.[0-9]+\\.[0-9]+$",
1572
- "description":"Semantic version of the contract (e.g., '2.1.0')"
1573
- },
1574
- "status":{
1575
- "type":"string",
1576
- "enum":["stable","deprecated","experimental"],
1577
- "default":"stable",
1578
- "description":"Contract status: stable (production-ready), deprecated (being phased out), experimental (testing)"
1579
- },
1580
- "compatibility":{
1581
- "$ref":"#/$defs/compatibilityConfig",
1582
- "description":"Compatibility information for this contract version"
1583
- },
1584
- "deprecation":{
1585
- "$ref":"#/$defs/deprecationConfig",
1586
- "description":"Deprecation information if status is 'deprecated'"
1587
- }
1588
- },
1589
- "additionalProperties":false
1590
- }
1591
- },
1592
- "additionalProperties":false
1593
- }
1594
-
1
+ {
2
+ "$schema":"https://json-schema.org/draft/2020-12/schema",
3
+ "$id":"https://raw.githubusercontent.com/esystemsdev/aifabrix-builder/refs/heads/main/lib/schema/external-datasource.schema.json",
4
+ "title":"External Data Source",
5
+ "description":"Configuration for AI Fabrix ExternalDataSource entities. Includes metadata schema, data dimensions, transformation mappings, OpenAPI/MCP exposure, execution logic, and sync behavior.",
6
+ "metadata":{
7
+ "key":"external-datasource-schema",
8
+ "name":"External Data Source Configuration Schema",
9
+ "description":"JSON schema for validating ExternalDataSource configuration files",
10
+ "version":"2.1.0",
11
+ "type":"schema",
12
+ "category":"integration",
13
+ "author":"AI Fabrix Team",
14
+ "createdAt":"2024-01-01T00:00:00Z",
15
+ "updatedAt":"2026-01-19T00:00:00Z",
16
+ "compatibility":{
17
+ "minVersion":"1.0.0",
18
+ "maxVersion":"3.0.0",
19
+ "deprecated":false
20
+ },
21
+ "tags":[
22
+ "schema",
23
+ "external-datasource",
24
+ "dataplane",
25
+ "integration",
26
+ "validation",
27
+ "cip",
28
+ "execution"
29
+ ],
30
+ "dependencies":[
31
+ "external-system.schema.json"
32
+ ],
33
+ "changelog":[
34
+ {
35
+ "version":"1.0.0",
36
+ "date":"2024-01-01T00:00:00Z",
37
+ "changes":[
38
+ "Initial schema for External Data Sources",
39
+ "Added fieldMappings DSL with transformation expressions",
40
+ "Added metadataSchema validation",
41
+ "Added exposed fields configuration for MCP/OpenAPI",
42
+ "Added ABAC accessFields validation",
43
+ "Added OpenAPI and MCP operation configurations",
44
+ "Added sync configuration and validation rules"
45
+ ],
46
+ "breaking":false
47
+ },
48
+ {
49
+ "version":"1.1.0",
50
+ "date":"2025-01-01T00:00:00Z",
51
+ "changes":[
52
+ "Added execution.engine with CIP and Python execution modes",
53
+ "Introduced CIP definition ($defs.cipDefinition and step types)",
54
+ "Added capabilities for list/get/create/update/delete",
55
+ "Extended fieldMappings with semantic metadata for AI",
56
+ "Added exposure profiles for different AI/analytics views",
57
+ "Added data quality rules, indexing, and AI context hints"
58
+ ],
59
+ "breaking":false
60
+ },
61
+ {
62
+ "version":"1.2.0",
63
+ "date":"2026-01-07T00:00:00Z",
64
+ "changes":[
65
+ "Added config.abac section with crossSystemSql and crossSystemJson properties",
66
+ "Added support for unified filter model (JSON and SQL formats) in ABAC cross-system configuration",
67
+ "Updated CIP filter expression to support both SQL and JSON formats"
68
+ ],
69
+ "breaking":false
70
+ },
71
+ {
72
+ "version":"2.0.0",
73
+ "date":"2026-01-18T00:00:00Z",
74
+ "changes":[
75
+ "BREAKING: Renamed fieldMappings.accessFields (array) to fieldMappings.dimensions (object)",
76
+ "BREAKING: Renamed fieldMappings.fields to fieldMappings.attributes",
77
+ "BREAKING: Renamed entityKey to entityType",
78
+ "BREAKING: Renamed exposed.fields to exposed.attributes",
79
+ "Added indexed property to attribute definitions for database index creation",
80
+ "Updated descriptions to reflect dimension-first data model approach"
81
+ ],
82
+ "breaking":true
83
+ },
84
+ {
85
+ "version":"2.1.0",
86
+ "date":"2026-01-19T00:00:00Z",
87
+ "changes":[
88
+ "Added error semantics (onError object) to all CIP step types with error classification, retry, compensation, and failPipeline",
89
+ "Added idempotency configuration to CIP definition for execution-level replay guarantees",
90
+ "Added lineage configuration to field mappings for field-level explainability and compliance",
91
+ "Added contract versioning configuration to datasource root for CI/CD safety and agent stability"
92
+ ],
93
+ "breaking":false
94
+ }
95
+ ]
96
+ },
97
+ "type":"object",
98
+ "required":[
99
+ "key",
100
+ "displayName",
101
+ "systemKey",
102
+ "entityType",
103
+ "resourceType",
104
+ "fieldMappings"
105
+ ],
106
+ "properties":{
107
+ "key":{
108
+ "type":"string",
109
+ "pattern":"^[a-z0-9-]+$",
110
+ "minLength":3,
111
+ "description":"Unique key of data source (e.g. 'hubspot-deal')."
112
+ },
113
+ "displayName":{
114
+ "type":"string",
115
+ "minLength":1
116
+ },
117
+ "description":{
118
+ "type":"string"
119
+ },
120
+ "enabled":{
121
+ "type":"boolean",
122
+ "default":true
123
+ },
124
+ "systemKey":{
125
+ "type":"string",
126
+ "pattern":"^[a-z0-9-]+$",
127
+ "description":"Must match ExternalSystem.key"
128
+ },
129
+ "entityType":{
130
+ "type":"string",
131
+ "enum":["document-storage","documentStorage","vector-store","vectorStore","record-storage","recordStorage","message-service","messageService","none"],
132
+ "description":"Entity type identifier. Defines storage semantics and behavior: 'document-storage' or 'documentStorage' (creates document storage service with vector-storage), 'vector-store' or 'vectorStore' (external vector storage system), 'record-storage' or 'recordStorage' (creates record-based system with metadata sync and access rights), 'message-service' or 'messageService' (message service for notifications - Slack, Teams, Email), 'none' (uses external system data directly or connects other data sources). Required field. Supports both camelCase and kebab-case formats for backward compatibility."
133
+ },
134
+ "resourceType":{
135
+ "type":"string",
136
+ "pattern":"^[a-z0-9-]+$",
137
+ "default":"document",
138
+ "description":"Resource type for the datasource. Represents business entity classification. Common values: customer, contact, person, document, deal, record. Any lowercase alphanumeric string with hyphens is allowed."
139
+ },
140
+ "version":{
141
+ "type":"string",
142
+ "pattern":"^[0-9]+\\.[0-9]+\\.[0-9]+$",
143
+ "default":"1.0.0"
144
+ },
145
+ "metadataSchema":{
146
+ "type":"object",
147
+ "description":"Subset of JSON Schema used to validate raw input metadata.",
148
+ "additionalProperties":true
149
+ },
150
+ "fieldMappings":{
151
+ "type":"object",
152
+ "description":"Transformation rules and data dimensions. Maps canonical dimensions to system attributes.",
153
+ "required":[
154
+ "dimensions",
155
+ "attributes"
156
+ ],
157
+ "properties":{
158
+ "dimensions":{
159
+ "type":"object",
160
+ "description":"Data dimensions mapping. Key = dimension key (from Dimension Catalog), Value = attribute path (e.g., 'metadata.department').",
161
+ "additionalProperties":{
162
+ "type":"string",
163
+ "pattern":"^[a-zA-Z0-9_.]+$"
164
+ },
165
+ "minProperties":0
166
+ },
167
+ "attributes":{
168
+ "type":"object",
169
+ "description":"Key = normalized attribute name. Value = transformation expression + type.",
170
+ "additionalProperties":{
171
+ "type":"object",
172
+ "required":[
173
+ "expression",
174
+ "type"
175
+ ],
176
+ "properties":{
177
+ "expression":{
178
+ "type":"string",
179
+ "description":"Pipe-based DSL expression: '{{raw.path}} | toLower | trim' or record reference: 'record_ref:customer'.",
180
+ "pattern":"^\\s*((\\{\\{[^}]+\\}\\}(\\s*\\|\\s*[a-zA-Z0-9_]+(\\([^)]*\\))?)*)|(record_ref:[a-z0-9-]+))\\s*$"
181
+ },
182
+ "type":{
183
+ "type":"string",
184
+ "enum":[
185
+ "string",
186
+ "number",
187
+ "integer",
188
+ "boolean",
189
+ "array",
190
+ "object"
191
+ ]
192
+ },
193
+ "indexed":{
194
+ "type":"boolean",
195
+ "default":false,
196
+ "description":"If true, creates database index for fast search/filtering. Dimensions are automatically indexed."
197
+ },
198
+ "description":{
199
+ "type":"string",
200
+ "description":"Technical description of the normalized field."
201
+ },
202
+ "required":{
203
+ "type":"boolean"
204
+ },
205
+ "semantic":{
206
+ "type":"object",
207
+ "description":"Semantic metadata for AI agents and schema generation.",
208
+ "properties":{
209
+ "title":{
210
+ "type":"string",
211
+ "description":"Human-friendly label for this field."
212
+ },
213
+ "description":{
214
+ "type":"string",
215
+ "description":"Business-level description of the field and how it is used."
216
+ },
217
+ "example":{
218
+ "description":"Example value for this field.",
219
+ "type":["string","number","boolean","object","array","null"]
220
+ },
221
+ "categories":{
222
+ "type":"array",
223
+ "description":"Logical categories/tags (e.g. ['sales', 'revenue']).",
224
+ "items":{
225
+ "type":"string"
226
+ }
227
+ },
228
+ "synonyms":{
229
+ "type":"array",
230
+ "description":"Optional list of synonyms used in natural language (e.g. 'opportunity', 'sales-case').",
231
+ "items":{
232
+ "type":"string"
233
+ }
234
+ }
235
+ },
236
+ "additionalProperties":false
237
+ },
238
+ "lineage":{
239
+ "$ref":"#/$defs/fieldMappingLineage"
240
+ }
241
+ }
242
+ }
243
+ }
244
+ }
245
+ },
246
+ "exposed":{
247
+ "type":"object",
248
+ "description":"Defines which normalized fields are exposed through MCP/OpenAPI. Ensures ISO27001-safe output and predictable AI model behavior.",
249
+ "properties":{
250
+ "attributes":{
251
+ "type":"array",
252
+ "description":"List of normalized attributes (from fieldMappings.attributes keys) that the MCP/OpenAPI layers will return.",
253
+ "items":{
254
+ "type":"string",
255
+ "pattern":"^[a-zA-Z0-9_]+$"
256
+ },
257
+ "minItems":1,
258
+ "uniqueItems":true
259
+ },
260
+ "omit":{
261
+ "type":"array",
262
+ "description":"Fields that exist in normalized metadata but must NEVER be exposed by MCP/OpenAPI (e.g., secrets, internal IDs). Overrides 'fields'.",
263
+ "items":{
264
+ "type":"string",
265
+ "pattern":"^[a-zA-Z0-9_]+$"
266
+ },
267
+ "uniqueItems":true
268
+ },
269
+ "groups":{
270
+ "type":"object",
271
+ "description":"Optional logical grouping for OpenAPI/MCP schema generation (e.g., 'default', 'analytics', 'light').",
272
+ "additionalProperties":{
273
+ "type":"array",
274
+ "items":{
275
+ "type":"string",
276
+ "pattern":"^[a-zA-Z0-9_]+$"
277
+ },
278
+ "minItems":1,
279
+ "uniqueItems":true
280
+ }
281
+ },
282
+ "readonly":{
283
+ "type":"array",
284
+ "description":"Fields exposed via MCP/OpenAPI but cannot be modified via write operations.",
285
+ "items":{
286
+ "type":"string",
287
+ "pattern":"^[a-zA-Z0-9_]+$"
288
+ },
289
+ "uniqueItems":true
290
+ },
291
+ "description":{
292
+ "type":"string",
293
+ "description":"Human-readable description of the exposure model for this data source."
294
+ },
295
+ "profiles":{
296
+ "type":"object",
297
+ "description":"Named exposure profiles mapping profile keys to lists of normalized field names. Useful for AI-light, AI-embedding, analytics, etc.",
298
+ "additionalProperties":{
299
+ "type":"array",
300
+ "items":{
301
+ "type":"string",
302
+ "pattern":"^[a-zA-Z0-9_]+$"
303
+ },
304
+ "minItems":1,
305
+ "uniqueItems":true
306
+ }
307
+ }
308
+ },
309
+ "required":[
310
+ "attributes"
311
+ ],
312
+ "additionalProperties":false
313
+ },
314
+ "sync":{
315
+ "type":"object",
316
+ "description":"Record synchronization rules.",
317
+ "properties":{
318
+ "mode":{
319
+ "type":"string",
320
+ "enum":[
321
+ "pull",
322
+ "push",
323
+ "bidirectional"
324
+ ],
325
+ "default":"pull"
326
+ },
327
+ "schedule":{
328
+ "type":"string"
329
+ },
330
+ "batchSize":{
331
+ "type":"integer",
332
+ "minimum":1,
333
+ "maximum":10000,
334
+ "default":500
335
+ },
336
+ "maxParallelRequests":{
337
+ "type":"integer",
338
+ "minimum":1,
339
+ "maximum":50,
340
+ "default":5
341
+ }
342
+ }
343
+ },
344
+ "openapi":{
345
+ "type":"object",
346
+ "description":"OpenAPI-driven connector configuration. Only includes endpoints selected during wizard onboarding. To add more endpoints, run the wizard again.",
347
+ "properties":{
348
+ "enabled":{
349
+ "type":"boolean",
350
+ "default":false
351
+ },
352
+ "documentKey":{
353
+ "type":"string"
354
+ },
355
+ "baseUrl":{
356
+ "type":"string",
357
+ "pattern":"^(http|https)://"
358
+ },
359
+ "resourcePath":{
360
+ "type":"string"
361
+ },
362
+ "operations":{
363
+ "type":"object",
364
+ "description":"Selected operations from the OpenAPI document. Only endpoints chosen during wizard onboarding are included.",
365
+ "properties":{
366
+ "list":{
367
+ "type":"object",
368
+ "required":[
369
+ "operationId"
370
+ ],
371
+ "properties":{
372
+ "operationId":{
373
+ "type":"string"
374
+ },
375
+ "method":{
376
+ "type":"string",
377
+ "enum":[
378
+ "GET",
379
+ "POST"
380
+ ]
381
+ },
382
+ "path":{
383
+ "type":"string"
384
+ }
385
+ }
386
+ },
387
+ "get":{
388
+ "type":"object",
389
+ "required":[
390
+ "operationId"
391
+ ],
392
+ "properties":{
393
+ "operationId":{
394
+ "type":"string"
395
+ },
396
+ "method":{
397
+ "type":"string",
398
+ "enum":[
399
+ "GET"
400
+ ]
401
+ },
402
+ "path":{
403
+ "type":"string"
404
+ }
405
+ }
406
+ },
407
+ "create":{
408
+ "type":"object",
409
+ "properties":{
410
+ "operationId":{
411
+ "type":"string"
412
+ },
413
+ "method":{
414
+ "type":"string",
415
+ "enum":[
416
+ "POST",
417
+ "PUT"
418
+ ]
419
+ },
420
+ "path":{
421
+ "type":"string"
422
+ }
423
+ }
424
+ },
425
+ "update":{
426
+ "type":"object",
427
+ "properties":{
428
+ "operationId":{
429
+ "type":"string"
430
+ },
431
+ "method":{
432
+ "type":"string",
433
+ "enum":[
434
+ "PATCH",
435
+ "PUT",
436
+ "POST"
437
+ ]
438
+ },
439
+ "path":{
440
+ "type":"string"
441
+ }
442
+ }
443
+ },
444
+ "delete":{
445
+ "type":"object",
446
+ "properties":{
447
+ "operationId":{
448
+ "type":"string"
449
+ },
450
+ "method":{
451
+ "type":"string",
452
+ "enum":[
453
+ "DELETE"
454
+ ]
455
+ },
456
+ "path":{
457
+ "type":"string"
458
+ }
459
+ }
460
+ }
461
+ }
462
+ },
463
+ "autoRbac":{
464
+ "type":"boolean",
465
+ "default":false,
466
+ "description":"Generates <system>.<entity>.<action> RBAC permissions."
467
+ }
468
+ }
469
+ },
470
+ "validation":{
471
+ "type":"object",
472
+ "description":"Advanced validation rules applied before saving normalized metadata.",
473
+ "properties":{
474
+ "repeatingValues":{
475
+ "type":"array",
476
+ "items":{
477
+ "type":"object",
478
+ "required":[
479
+ "field",
480
+ "scope",
481
+ "strategy"
482
+ ],
483
+ "properties":{
484
+ "field":{
485
+ "type":"string",
486
+ "pattern":"^[a-zA-Z0-9_]+$"
487
+ },
488
+ "scope":{
489
+ "type":"array",
490
+ "items":{
491
+ "type":"string"
492
+ }
493
+ },
494
+ "strategy":{
495
+ "type":"string",
496
+ "enum":[
497
+ "reject",
498
+ "latest-wins",
499
+ "first-wins",
500
+ "merge"
501
+ ]
502
+ },
503
+ "mergeFields":{
504
+ "type":"array",
505
+ "items":{
506
+ "type":"string"
507
+ }
508
+ }
509
+ }
510
+ }
511
+ }
512
+ }
513
+ },
514
+ "quality":{
515
+ "type":"object",
516
+ "description":"Data quality rules applied after mapping but before persistence and AI exposure.",
517
+ "properties":{
518
+ "rejectIf":{
519
+ "type":"array",
520
+ "description":"List of conditions that cause a record to be rejected.",
521
+ "items":{
522
+ "type":"object",
523
+ "required":[
524
+ "field",
525
+ "operator"
526
+ ],
527
+ "properties":{
528
+ "field":{
529
+ "type":"string",
530
+ "pattern":"^[a-zA-Z0-9_]+$",
531
+ "description":"Normalized field name to evaluate."
532
+ },
533
+ "operator":{
534
+ "type":"string",
535
+ "enum":[
536
+ "missing",
537
+ "null",
538
+ "empty",
539
+ "invalidRef",
540
+ "lessThan",
541
+ "greaterThan",
542
+ "equal",
543
+ "notEqual"
544
+ ]
545
+ },
546
+ "value":{
547
+ "description":"Comparison value for applicable operators.",
548
+ "type":["string","number","boolean","null"]
549
+ },
550
+ "message":{
551
+ "type":"string",
552
+ "description":"Optional human-readable reason for rejection."
553
+ }
554
+ },
555
+ "additionalProperties":false
556
+ }
557
+ }
558
+ },
559
+ "additionalProperties":false
560
+ },
561
+ "indexing":{
562
+ "type":"object",
563
+ "description":"Indexing and embedding strategy for this datasource.",
564
+ "properties":{
565
+ "embedding":{
566
+ "type":"array",
567
+ "description":"List of normalized fields whose values should be concatenated and embedded for vector search.",
568
+ "items":{
569
+ "type":"string",
570
+ "pattern":"^[a-zA-Z0-9_]+$"
571
+ },
572
+ "minItems":1,
573
+ "uniqueItems":true
574
+ },
575
+ "uniqueKey":{
576
+ "type":"string",
577
+ "pattern":"^[a-zA-Z0-9_]+$",
578
+ "description":"Normalized field used as unique identifier for deduplication and upserts."
579
+ },
580
+ "dedupeStrategy":{
581
+ "type":"string",
582
+ "enum":[
583
+ "reject",
584
+ "latest-wins",
585
+ "first-wins",
586
+ "merge"
587
+ ],
588
+ "default":"latest-wins",
589
+ "description":"Strategy to apply when multiple records share the same uniqueKey."
590
+ }
591
+ },
592
+ "additionalProperties":false
593
+ },
594
+ "context":{
595
+ "type":"object",
596
+ "description":"Natural-language and semantic hints for AI agents.",
597
+ "properties":{
598
+ "semanticTags":{
599
+ "type":"array",
600
+ "description":"High-level tags describing this datasource (e.g. ['crm', 'customer-data', 'deal']).",
601
+ "items":{
602
+ "type":"string"
603
+ }
604
+ },
605
+ "synonyms":{
606
+ "type":"array",
607
+ "description":"Terms users might use in natural language to refer to this datasource or its primary entity.",
608
+ "items":{
609
+ "type":"string"
610
+ }
611
+ },
612
+ "naturalLanguageHints":{
613
+ "type":"array",
614
+ "description":"Free-form hints for LLMs on how to interpret or use this datasource.",
615
+ "items":{
616
+ "type":"string"
617
+ }
618
+ }
619
+ },
620
+ "additionalProperties":false
621
+ },
622
+ "documentStorage":{
623
+ "type":"object",
624
+ "description":"Document storage configuration (optional, enables vector storage). Validated against type/document-storage.json schema.",
625
+ "properties":{
626
+ "enabled":{
627
+ "type":"boolean",
628
+ "default":true
629
+ },
630
+ "twoPhaseSync":{
631
+ "type":"boolean",
632
+ "default":true,
633
+ "description":"Enable two-phase sync pattern. When true: validates metadata first (quality rules, comparison with DocumentRecords), then fetches binaries via CIP for changed/new documents. When false: fetches binaries directly without metadata validation phase (single-phase sync). Note: Files are never synced back to external systems (one-way sync only: external → dataplane)."
634
+ },
635
+ "binaryOperationRef":{
636
+ "type":"string",
637
+ "default":"get",
638
+ "description":"CIP operation name for binary document retrieval. Must exist in execution.cip.operations. Defaults to 'get' operation."
639
+ },
640
+ "responseType":{
641
+ "type":"string",
642
+ "enum":[
643
+ "binary",
644
+ "base64",
645
+ "json"
646
+ ],
647
+ "default":"binary",
648
+ "description":"Expected response type from CIP operation. 'binary' for raw binary data, 'base64' for base64-encoded data, 'json' for JSON response with binary field."
649
+ },
650
+ "binaryField":{
651
+ "type":"string",
652
+ "description":"Field name containing binary data if responseType is 'json' or 'base64'. Required when responseType is not 'binary'."
653
+ },
654
+ "parameterMapping":{
655
+ "type":"object",
656
+ "additionalProperties":{
657
+ "type":"string"
658
+ },
659
+ "description":"Map metadata record fields to CIP operation parameters. Example: {\"fileId\": \"{{key}}\", \"downloadUrl\": \"{{metadata.downloadUrl}}\"}"
660
+ },
661
+ "processing":{
662
+ "type":"object",
663
+ "properties":{
664
+ "fileStoragePath":{
665
+ "type":"string",
666
+ "default":"/data/documents"
667
+ },
668
+ "aiValidation":{
669
+ "type":"object"
670
+ },
671
+ "spacyEnrichment":{
672
+ "type":"object"
673
+ },
674
+ "notifications":{
675
+ "type":"object"
676
+ }
677
+ },
678
+ "additionalProperties":false
679
+ },
680
+ "credentialId":{
681
+ "type":"string"
682
+ }
683
+ },
684
+ "required":[
685
+ "enabled"
686
+ ],
687
+ "additionalProperties":false
688
+ },
689
+ "portalInput":{
690
+ "type":"array",
691
+ "description":"Optional UI metadata definition for the AI Fabrix portal.",
692
+ "items":{
693
+ "type":"object",
694
+ "required":[
695
+ "name",
696
+ "field",
697
+ "label"
698
+ ],
699
+ "properties":{
700
+ "name":{
701
+ "type":"string"
702
+ },
703
+ "field":{
704
+ "type":"string",
705
+ "enum":[
706
+ "text",
707
+ "textarea",
708
+ "select",
709
+ "json",
710
+ "boolean",
711
+ "number"
712
+ ]
713
+ },
714
+ "label":{
715
+ "type":"string"
716
+ },
717
+ "placeholder":{
718
+ "type":"string"
719
+ },
720
+ "options":{
721
+ "type":"array",
722
+ "items":{
723
+ "type":"string"
724
+ }
725
+ },
726
+ "masked":{
727
+ "type":"boolean"
728
+ },
729
+ "validation":{
730
+ "type":"object",
731
+ "properties":{
732
+ "minLength":{
733
+ "type":"integer"
734
+ },
735
+ "maxLength":{
736
+ "type":"integer"
737
+ },
738
+ "pattern":{
739
+ "type":"string"
740
+ },
741
+ "required":{
742
+ "type":"boolean"
743
+ }
744
+ }
745
+ }
746
+ }
747
+ }
748
+ },
749
+ "testPayload":{
750
+ "type":"object",
751
+ "description":"Test payload configuration for unit and integration testing",
752
+ "properties":{
753
+ "payloadTemplate":{
754
+ "type":"object",
755
+ "description":"Sample payload matching the expected API response structure. Used for testing field mappings and metadata schema validation.",
756
+ "additionalProperties":true
757
+ },
758
+ "expectedResult":{
759
+ "type":"object",
760
+ "description":"Expected normalized result after field mapping transformations (optional, for validation)",
761
+ "additionalProperties":true
762
+ }
763
+ },
764
+ "additionalProperties":false
765
+ },
766
+ "capabilities":{
767
+ "type":"object",
768
+ "description":"Declares which logical operations are supported by this datasource.",
769
+ "properties":{
770
+ "list":{
771
+ "type":"boolean",
772
+ "default":true
773
+ },
774
+ "get":{
775
+ "type":"boolean",
776
+ "default":false
777
+ },
778
+ "create":{
779
+ "type":"boolean",
780
+ "default":false
781
+ },
782
+ "update":{
783
+ "type":"boolean",
784
+ "default":false
785
+ },
786
+ "delete":{
787
+ "type":"boolean",
788
+ "default":false
789
+ }
790
+ },
791
+ "additionalProperties":false
792
+ },
793
+ "execution":{
794
+ "type":"object",
795
+ "description":"Execution engine configuration for this datasource (CIP or Python). CIP is the native declarative mode.",
796
+ "required":[
797
+ "engine"
798
+ ],
799
+ "properties":{
800
+ "engine":{
801
+ "type":"string",
802
+ "enum":[
803
+ "cip",
804
+ "python"
805
+ ],
806
+ "description":"Execution engine. 'cip' for declarative pipelines, 'python' for custom handlers."
807
+ },
808
+ "cip":{
809
+ "$ref":"#/$defs/cipDefinition"
810
+ },
811
+ "python":{
812
+ "$ref":"#/$defs/pythonExecution"
813
+ }
814
+ },
815
+ "additionalProperties":false
816
+ },
817
+ "config":{
818
+ "type":"object",
819
+ "description":"Additional configuration for this datasource, including ABAC settings, MCP contracts, and other metadata.",
820
+ "properties":{
821
+ "abac":{
822
+ "type":"object",
823
+ "description":"Attribute-Based Access Control (ABAC) configuration for this datasource.",
824
+ "properties":{
825
+ "dimensions":{
826
+ "type":"object",
827
+ "description":"Data dimensions mapping. Key = dimension key (from Dimension Catalog), Value = attribute path. Overrides fieldMappings.dimensions if specified.",
828
+ "additionalProperties":{
829
+ "type":"string",
830
+ "pattern":"^[a-zA-Z0-9_.]+$"
831
+ }
832
+ },
833
+ "crossSystemSql":{
834
+ "type":"string",
835
+ "description":"Cross-system ABAC filter expression in SQL format (advanced, for developers). Example: 'hubspot-companies.country = user.country AND hubspot-companies.revenue >= 1000000'"
836
+ },
837
+ "crossSystemJson":{
838
+ "type":"object",
839
+ "description":"Cross-system ABAC filter expression in JSON format (simple, for UI). Example: {'hubspot-companies.country': {'eq': 'user.country'}, 'hubspot-companies.revenue': {'gte': 1000000}}",
840
+ "additionalProperties":{
841
+ "type":"object",
842
+ "properties":{
843
+ "eq":{"type":["string","number","boolean"]},
844
+ "ne":{"type":["string","number","boolean"]},
845
+ "gt":{"type":["string","number"]},
846
+ "lt":{"type":["string","number"]},
847
+ "gte":{"type":["string","number"]},
848
+ "lte":{"type":["string","number"]},
849
+ "in":{"type":"array"},
850
+ "nin":{"type":"array"},
851
+ "contains":{"type":"string"},
852
+ "like":{"type":"string"},
853
+ "isNull":{"type":"null"},
854
+ "isNotNull":{"type":"null"}
855
+ }
856
+ }
857
+ }
858
+ },
859
+ "additionalProperties":false
860
+ }
861
+ },
862
+ "additionalProperties":true
863
+ },
864
+ "contract":{
865
+ "$ref":"#/$defs/contractConfig"
866
+ }
867
+ },
868
+ "$defs":{
869
+ "pythonExecution":{
870
+ "type":"object",
871
+ "description":"Python-based execution config for advanced/custom use cases.",
872
+ "properties":{
873
+ "entrypoint":{
874
+ "type":"string",
875
+ "description":"Python callable in module notation, e.g. 'handlers.hubspot_deal.list_records'."
876
+ },
877
+ "codeRef":{
878
+ "type":"string",
879
+ "description":"Optional reference to external code bundle identifier or embedded block key."
880
+ }
881
+ },
882
+ "required":[
883
+ "entrypoint"
884
+ ],
885
+ "additionalProperties":false
886
+ },
887
+ "cipLineage":{
888
+ "type":"object",
889
+ "description":"Optional lineage metadata for a CIP operation or step.",
890
+ "properties":{
891
+ "producer":{
892
+ "type":"string",
893
+ "description":"Upstream system or connector name (e.g. 'hubspot')."
894
+ },
895
+ "entity":{
896
+ "type":"string",
897
+ "description":"Upstream entity or resource (e.g. 'deal')."
898
+ },
899
+ "operation":{
900
+ "type":"string",
901
+ "description":"Logical operation (e.g. 'list', 'get', 'sync-job-1')."
902
+ }
903
+ },
904
+ "additionalProperties":false
905
+ },
906
+ "cipDefinition":{
907
+ "type":"object",
908
+ "description":"Composable Integration Pipeline (CIP) definition for this datasource.",
909
+ "required":[
910
+ "version",
911
+ "operations"
912
+ ],
913
+ "properties":{
914
+ "version":{
915
+ "type":"string",
916
+ "enum":[
917
+ "1.0"
918
+ ]
919
+ },
920
+ "inputType":{
921
+ "type":"string",
922
+ "const":"void",
923
+ "description":"Logical input type of the pipeline. Reserved for future extensions."
924
+ },
925
+ "outputType":{
926
+ "type":"string",
927
+ "const":"records",
928
+ "description":"Logical output type of the pipeline."
929
+ },
930
+ "operations":{
931
+ "type":"object",
932
+ "description":"Logical operations (list/get/create/update/delete) implemented for this datasource.",
933
+ "properties":{
934
+ "list":{
935
+ "$ref":"#/$defs/cipOperation"
936
+ },
937
+ "get":{
938
+ "$ref":"#/$defs/cipOperation"
939
+ },
940
+ "create":{
941
+ "$ref":"#/$defs/cipOperation"
942
+ },
943
+ "update":{
944
+ "$ref":"#/$defs/cipOperation"
945
+ },
946
+ "delete":{
947
+ "$ref":"#/$defs/cipOperation"
948
+ }
949
+ },
950
+ "additionalProperties":false
951
+ },
952
+ "idempotency":{
953
+ "$ref":"#/$defs/idempotencyConfig"
954
+ }
955
+ },
956
+ "additionalProperties":false
957
+ },
958
+ "cipOperation":{
959
+ "type":"object",
960
+ "required":[
961
+ "steps"
962
+ ],
963
+ "properties":{
964
+ "description":{
965
+ "type":"string"
966
+ },
967
+ "enabled":{
968
+ "type":"boolean",
969
+ "default":true
970
+ },
971
+ "lineage":{
972
+ "$ref":"#/$defs/cipLineage"
973
+ },
974
+ "steps":{
975
+ "type":"array",
976
+ "minItems":1,
977
+ "items":{
978
+ "$ref":"#/$defs/cipStep"
979
+ }
980
+ }
981
+ },
982
+ "additionalProperties":false
983
+ },
984
+ "cipStep":{
985
+ "description":"One step in the CIP pipeline.",
986
+ "oneOf":[
987
+ {
988
+ "$ref":"#/$defs/cipStepFetch"
989
+ },
990
+ {
991
+ "$ref":"#/$defs/cipStepPaginate"
992
+ },
993
+ {
994
+ "$ref":"#/$defs/cipStepMap"
995
+ },
996
+ {
997
+ "$ref":"#/$defs/cipStepFilter"
998
+ },
999
+ {
1000
+ "$ref":"#/$defs/cipStepOutput"
1001
+ },
1002
+ {
1003
+ "$ref":"#/$defs/cipStepPythonInline"
1004
+ }
1005
+ ]
1006
+ },
1007
+ "cipStepFetch":{
1008
+ "type":"object",
1009
+ "required":[
1010
+ "fetch"
1011
+ ],
1012
+ "properties":{
1013
+ "fetch":{
1014
+ "type":"object",
1015
+ "description":"Fetch raw records from an upstream system.",
1016
+ "required":[
1017
+ "source"
1018
+ ],
1019
+ "properties":{
1020
+ "source":{
1021
+ "type":"string",
1022
+ "enum":[
1023
+ "openapi",
1024
+ "http"
1025
+ ],
1026
+ "description":"If 'openapi', refer to openapi.operations.*. If 'http', use explicit method/path."
1027
+ },
1028
+ "openapiRef":{
1029
+ "type":"string",
1030
+ "enum":[
1031
+ "list",
1032
+ "get",
1033
+ "create",
1034
+ "update",
1035
+ "delete"
1036
+ ],
1037
+ "description":"Reference to openapi.operations.<openapiRef> entry."
1038
+ },
1039
+ "operationId":{
1040
+ "type":"string",
1041
+ "description":"OpenAPI operationId to call when source='openapi'. Optional when openapiRef is used."
1042
+ },
1043
+ "method":{
1044
+ "type":"string",
1045
+ "enum":[
1046
+ "GET",
1047
+ "POST",
1048
+ "PUT",
1049
+ "PATCH",
1050
+ "DELETE"
1051
+ ],
1052
+ "description":"Explicit HTTP method when source='http'."
1053
+ },
1054
+ "path":{
1055
+ "type":"string",
1056
+ "description":"Explicit HTTP path when source='http'."
1057
+ },
1058
+ "query":{
1059
+ "type":"object",
1060
+ "description":"Static query parameters. ABAC/runtime can still override or append at runtime.",
1061
+ "additionalProperties":{
1062
+ "type":[
1063
+ "string",
1064
+ "number",
1065
+ "boolean",
1066
+ "null"
1067
+ ]
1068
+ }
1069
+ },
1070
+ "bodyTemplate":{
1071
+ "description":"Optional static request body or template for POST/PUT/PATCH fetches.",
1072
+ "type":["object","string","null"]
1073
+ },
1074
+ "headers":{
1075
+ "type":"object",
1076
+ "description":"Optional HTTP headers for the request.",
1077
+ "additionalProperties":{
1078
+ "type":"string"
1079
+ }
1080
+ }
1081
+ },
1082
+ "allOf":[
1083
+ {
1084
+ "if":{
1085
+ "properties":{
1086
+ "source":{
1087
+ "const":"openapi"
1088
+ }
1089
+ }
1090
+ },
1091
+ "then":{
1092
+ "anyOf":[
1093
+ {
1094
+ "required":[
1095
+ "openapiRef"
1096
+ ]
1097
+ },
1098
+ {
1099
+ "required":[
1100
+ "operationId"
1101
+ ]
1102
+ }
1103
+ ]
1104
+ }
1105
+ },
1106
+ {
1107
+ "if":{
1108
+ "properties":{
1109
+ "source":{
1110
+ "const":"http"
1111
+ }
1112
+ }
1113
+ },
1114
+ "then":{
1115
+ "required":[
1116
+ "method",
1117
+ "path"
1118
+ ]
1119
+ }
1120
+ }
1121
+ ],
1122
+ "additionalProperties":false
1123
+ },
1124
+ "lineage":{
1125
+ "$ref":"#/$defs/cipLineage"
1126
+ },
1127
+ "stepId":{
1128
+ "type":"string",
1129
+ "pattern":"^[a-z0-9-_]+$",
1130
+ "description":"Unique identifier for this step within the operation"
1131
+ },
1132
+ "onError":{
1133
+ "$ref":"#/$defs/cipOnErrorConfig"
1134
+ }
1135
+ },
1136
+ "additionalProperties":false
1137
+ },
1138
+ "cipStepPaginate":{
1139
+ "type":"object",
1140
+ "required":[
1141
+ "paginate"
1142
+ ],
1143
+ "properties":{
1144
+ "paginate":{
1145
+ "type":"object",
1146
+ "description":"Pagination strategy applied to results from the preceding fetch.",
1147
+ "required":[
1148
+ "strategy"
1149
+ ],
1150
+ "properties":{
1151
+ "strategy":{
1152
+ "type":"string",
1153
+ "enum":[
1154
+ "cursor",
1155
+ "page",
1156
+ "offset",
1157
+ "none"
1158
+ ]
1159
+ },
1160
+ "cursorField":{
1161
+ "type":"string",
1162
+ "description":"Field in the response payload containing the cursor value (for 'cursor' strategy)."
1163
+ },
1164
+ "cursorParam":{
1165
+ "type":"string",
1166
+ "description":"Name of the query parameter used to send the cursor."
1167
+ },
1168
+ "pageParam":{
1169
+ "type":"string",
1170
+ "description":"Name of the query parameter used to send the page number (for 'page' strategy)."
1171
+ },
1172
+ "pageSizeParam":{
1173
+ "type":"string",
1174
+ "description":"Name of the query parameter used to send the page size."
1175
+ },
1176
+ "pageSize":{
1177
+ "type":"integer",
1178
+ "minimum":1,
1179
+ "description":"Default page size."
1180
+ },
1181
+ "maxPages":{
1182
+ "type":"integer",
1183
+ "minimum":1,
1184
+ "description":"Maximum number of pages to fetch."
1185
+ }
1186
+ },
1187
+ "additionalProperties":false
1188
+ },
1189
+ "lineage":{
1190
+ "$ref":"#/$defs/cipLineage"
1191
+ },
1192
+ "stepId":{
1193
+ "type":"string",
1194
+ "pattern":"^[a-z0-9-_]+$",
1195
+ "description":"Unique identifier for this step within the operation"
1196
+ },
1197
+ "onError":{
1198
+ "$ref":"#/$defs/cipOnErrorConfig"
1199
+ }
1200
+ },
1201
+ "additionalProperties":false
1202
+ },
1203
+ "cipStepMap":{
1204
+ "type":"object",
1205
+ "required":[
1206
+ "map"
1207
+ ],
1208
+ "properties":{
1209
+ "map":{
1210
+ "type":"object",
1211
+ "description":"Transform raw API records into the normalized metadata model.",
1212
+ "properties":{
1213
+ "inputPath":{
1214
+ "type":"string",
1215
+ "default":"$.items[*]",
1216
+ "description":"JSONPath-like expression to iterate over records in the upstream payload."
1217
+ },
1218
+ "useFieldMappings":{
1219
+ "type":"boolean",
1220
+ "default":true,
1221
+ "description":"If true, apply the datasource.fieldMappings.attributes expressions; no inline mapping needed."
1222
+ },
1223
+ "inline":{
1224
+ "type":"object",
1225
+ "description":"Optional inline mapping overrides { normalizedField -> expression }.",
1226
+ "additionalProperties":{
1227
+ "type":"string",
1228
+ "description":"Pipe-based DSL expression overriding fieldMappings expression."
1229
+ }
1230
+ }
1231
+ },
1232
+ "additionalProperties":false
1233
+ },
1234
+ "lineage":{
1235
+ "$ref":"#/$defs/cipLineage"
1236
+ },
1237
+ "stepId":{
1238
+ "type":"string",
1239
+ "pattern":"^[a-z0-9-_]+$",
1240
+ "description":"Unique identifier for this step within the operation"
1241
+ },
1242
+ "onError":{
1243
+ "$ref":"#/$defs/cipOnErrorConfig"
1244
+ }
1245
+ },
1246
+ "additionalProperties":false
1247
+ },
1248
+ "cipStepFilter":{
1249
+ "type":"object",
1250
+ "required":[
1251
+ "filter"
1252
+ ],
1253
+ "properties":{
1254
+ "filter":{
1255
+ "type":"object",
1256
+ "description":"Filter mapped records before output. Dimension enforcement is applied automatically based on fieldMappings.dimensions.",
1257
+ "properties":{
1258
+ "enforceAbac":{
1259
+ "type":"boolean",
1260
+ "default":true
1261
+ },
1262
+ "expression":{
1263
+ "oneOf":[
1264
+ {
1265
+ "type":"string",
1266
+ "description":"SQL filter expression (e.g., 'status = \"active\" AND revenue >= 1000000')"
1267
+ },
1268
+ {
1269
+ "type":"object",
1270
+ "description":"JSON filter expression (e.g., {'status': {'eq': 'active'}, 'revenue': {'gte': 1000000}})",
1271
+ "additionalProperties":{
1272
+ "type":"object"
1273
+ }
1274
+ }
1275
+ ],
1276
+ "description":"Optional additional filter expression. Supports both SQL and JSON formats."
1277
+ }
1278
+ },
1279
+ "additionalProperties":false
1280
+ },
1281
+ "lineage":{
1282
+ "$ref":"#/$defs/cipLineage"
1283
+ },
1284
+ "stepId":{
1285
+ "type":"string",
1286
+ "pattern":"^[a-z0-9-_]+$",
1287
+ "description":"Unique identifier for this step within the operation"
1288
+ },
1289
+ "onError":{
1290
+ "$ref":"#/$defs/cipOnErrorConfig"
1291
+ }
1292
+ },
1293
+ "additionalProperties":false
1294
+ },
1295
+ "cipStepOutput":{
1296
+ "type":"object",
1297
+ "required":[
1298
+ "output"
1299
+ ],
1300
+ "properties":{
1301
+ "output":{
1302
+ "type":"object",
1303
+ "description":"Finalize the CIP pipeline and emit records.",
1304
+ "properties":{
1305
+ "mode":{
1306
+ "type":"string",
1307
+ "enum":[
1308
+ "records"
1309
+ ],
1310
+ "default":"records"
1311
+ },
1312
+ "limit":{
1313
+ "type":"integer",
1314
+ "minimum":1,
1315
+ "description":"Optional soft limit for records returned by list operation; sync runner can override."
1316
+ },
1317
+ "includeConfidence":{
1318
+ "type":"boolean",
1319
+ "default":false,
1320
+ "description":"If true, the pipeline will compute a confidence score per record based on quality checks, mapping completeness, and source hints."
1321
+ }
1322
+ },
1323
+ "additionalProperties":false
1324
+ },
1325
+ "lineage":{
1326
+ "$ref":"#/$defs/cipLineage"
1327
+ },
1328
+ "stepId":{
1329
+ "type":"string",
1330
+ "pattern":"^[a-z0-9-_]+$",
1331
+ "description":"Unique identifier for this step within the operation"
1332
+ },
1333
+ "onError":{
1334
+ "$ref":"#/$defs/cipOnErrorConfig"
1335
+ }
1336
+ },
1337
+ "additionalProperties":false
1338
+ },
1339
+ "cipStepPythonInline":{
1340
+ "type":"object",
1341
+ "required":[
1342
+ "pythonInline"
1343
+ ],
1344
+ "properties":{
1345
+ "pythonInline":{
1346
+ "type":"object",
1347
+ "description":"Sandboxed inline Python transformation, operating on the current record batch.",
1348
+ "properties":{
1349
+ "code":{
1350
+ "type":"string",
1351
+ "description":"Small, pure function snippet; no imports, no IO, no network. Runtime must enforce sandboxing."
1352
+ }
1353
+ },
1354
+ "required":[
1355
+ "code"
1356
+ ],
1357
+ "additionalProperties":false
1358
+ },
1359
+ "lineage":{
1360
+ "$ref":"#/$defs/cipLineage"
1361
+ },
1362
+ "stepId":{
1363
+ "type":"string",
1364
+ "pattern":"^[a-z0-9-_]+$",
1365
+ "description":"Unique identifier for this step within the operation"
1366
+ },
1367
+ "onError":{
1368
+ "$ref":"#/$defs/cipOnErrorConfig"
1369
+ }
1370
+ },
1371
+ "additionalProperties":false
1372
+ },
1373
+ "cipRetryConfig":{
1374
+ "type":"object",
1375
+ "description":"Retry configuration for error handling.",
1376
+ "properties":{
1377
+ "attempts":{
1378
+ "type":"integer",
1379
+ "minimum":1,
1380
+ "maximum":10,
1381
+ "default":3,
1382
+ "description":"Maximum number of retry attempts"
1383
+ },
1384
+ "backoff":{
1385
+ "type":"string",
1386
+ "enum":["exponential","linear","fixed"],
1387
+ "default":"exponential",
1388
+ "description":"Backoff strategy: exponential (2^attempt), linear (attempt * base), fixed (constant)"
1389
+ },
1390
+ "maxDelaySeconds":{
1391
+ "type":"number",
1392
+ "minimum":1.0,
1393
+ "maximum":300.0,
1394
+ "default":60.0,
1395
+ "description":"Maximum delay between retries in seconds"
1396
+ }
1397
+ },
1398
+ "additionalProperties":false
1399
+ },
1400
+ "cipCompensationConfig":{
1401
+ "type":"object",
1402
+ "description":"Compensation (rollback) configuration for error handling.",
1403
+ "properties":{
1404
+ "enabled":{
1405
+ "type":"boolean",
1406
+ "default":false,
1407
+ "description":"Whether compensation is enabled"
1408
+ },
1409
+ "stepRef":{
1410
+ "type":"string",
1411
+ "pattern":"^[a-z0-9-_]+$",
1412
+ "description":"Reference to a previously executed step by stepId to execute as compensation"
1413
+ }
1414
+ },
1415
+ "additionalProperties":false
1416
+ },
1417
+ "cipOnErrorConfig":{
1418
+ "type":"object",
1419
+ "description":"Enhanced error handling configuration for CIP steps.",
1420
+ "required":["class"],
1421
+ "properties":{
1422
+ "class":{
1423
+ "type":"string",
1424
+ "enum":["transient","permanent","validation","auth"],
1425
+ "description":"Error classification: transient (retryable), permanent (non-retryable), validation (data validation failure), auth (authentication/authorization failure)"
1426
+ },
1427
+ "retry":{
1428
+ "$ref":"#/$defs/cipRetryConfig",
1429
+ "description":"Retry configuration (only applies when class='transient')"
1430
+ },
1431
+ "compensate":{
1432
+ "$ref":"#/$defs/cipCompensationConfig",
1433
+ "description":"Compensation (rollback) configuration"
1434
+ },
1435
+ "failPipeline":{
1436
+ "type":"boolean",
1437
+ "default":true,
1438
+ "description":"Whether step failure should fail the entire pipeline (false allows partial success)"
1439
+ }
1440
+ },
1441
+ "additionalProperties":false
1442
+ },
1443
+ "idempotencyConfig":{
1444
+ "type":"object",
1445
+ "description":"Idempotency configuration for CIP execution.",
1446
+ "required":["key"],
1447
+ "properties":{
1448
+ "key":{
1449
+ "type":"string",
1450
+ "description":"Idempotency key template. Supports variables: {{actor.userId}}, {{raw.requestId}}, {{correlationId}}, {{execution.operation}}, {{execution.datasourceId}}"
1451
+ },
1452
+ "scope":{
1453
+ "type":"string",
1454
+ "enum":["datasource","system"],
1455
+ "default":"datasource",
1456
+ "description":"Idempotency scope: datasource (same key within datasource) or system (same key across all datasources)"
1457
+ },
1458
+ "ttlSeconds":{
1459
+ "type":"integer",
1460
+ "minimum":1,
1461
+ "maximum":604800,
1462
+ "default":86400,
1463
+ "description":"Time to live for idempotency records in seconds (default: 86400 = 24 hours)"
1464
+ },
1465
+ "enforcement":{
1466
+ "type":"string",
1467
+ "enum":["strict","best-effort"],
1468
+ "default":"strict",
1469
+ "description":"Enforcement mode: strict (return previous result, no execution) or best-effort (warn + continue)"
1470
+ }
1471
+ },
1472
+ "additionalProperties":false
1473
+ },
1474
+ "transformationRule":{
1475
+ "type":"object",
1476
+ "description":"Transformation rule applied to a field.",
1477
+ "required":["type"],
1478
+ "properties":{
1479
+ "type":{
1480
+ "type":"string",
1481
+ "description":"Type of transformation: 'mapping', 'function', 'filter', 'aggregate'"
1482
+ },
1483
+ "function":{
1484
+ "type":"string",
1485
+ "description":"Function name (e.g., 'mapStatus', 'toLower', 'trim')"
1486
+ },
1487
+ "rule":{
1488
+ "type":"string",
1489
+ "description":"Human-readable rule description (e.g., 'open → OPEN, closed → CLOSED')"
1490
+ }
1491
+ },
1492
+ "additionalProperties":false
1493
+ },
1494
+ "fieldMappingLineage":{
1495
+ "type":"object",
1496
+ "description":"Lineage metadata for a field mapping.",
1497
+ "required":["sourceFields"],
1498
+ "properties":{
1499
+ "sourceFields":{
1500
+ "type":"array",
1501
+ "minItems":1,
1502
+ "items":{
1503
+ "type":"string"
1504
+ },
1505
+ "description":"Source field paths that contribute to this mapped field (e.g., ['raw.fields.state'])"
1506
+ },
1507
+ "transformations":{
1508
+ "type":"array",
1509
+ "items":{
1510
+ "$ref":"#/$defs/transformationRule"
1511
+ },
1512
+ "description":"List of transformations applied to source fields"
1513
+ },
1514
+ "confidence":{
1515
+ "type":"number",
1516
+ "minimum":0.0,
1517
+ "maximum":1.0,
1518
+ "default":1.0,
1519
+ "description":"Confidence score for this mapping (0.0-1.0). Default: 1.0 for deterministic transformations."
1520
+ }
1521
+ },
1522
+ "additionalProperties":false
1523
+ },
1524
+ "compatibilityConfig":{
1525
+ "type":"object",
1526
+ "description":"Compatibility configuration for contract versioning.",
1527
+ "properties":{
1528
+ "backwardCompatible":{
1529
+ "type":"boolean",
1530
+ "default":true,
1531
+ "description":"Whether this version is backward compatible with previous versions"
1532
+ },
1533
+ "breakingChanges":{
1534
+ "type":"array",
1535
+ "items":{
1536
+ "type":"string"
1537
+ },
1538
+ "description":"List of breaking change types: removeField, changeType, changeSemantics"
1539
+ }
1540
+ },
1541
+ "additionalProperties":false
1542
+ },
1543
+ "deprecationConfig":{
1544
+ "type":"object",
1545
+ "description":"Deprecation configuration for contract versioning.",
1546
+ "properties":{
1547
+ "sunsetDate":{
1548
+ "type":"string",
1549
+ "pattern":"^\\d{4}-\\d{2}-\\d{2}$",
1550
+ "description":"Date when this contract version will be sunset (YYYY-MM-DD)"
1551
+ },
1552
+ "replacedBy":{
1553
+ "type":"string",
1554
+ "description":"Contract name/version that replaces this deprecated version"
1555
+ }
1556
+ },
1557
+ "additionalProperties":false
1558
+ },
1559
+ "contractConfig":{
1560
+ "type":"object",
1561
+ "description":"Contract versioning configuration for datasource.",
1562
+ "required":["name","version"],
1563
+ "properties":{
1564
+ "name":{
1565
+ "type":"string",
1566
+ "pattern":"^[a-z0-9.-]+$",
1567
+ "description":"Contract identifier (e.g., 'jira.issue', separate from datasource key)"
1568
+ },
1569
+ "version":{
1570
+ "type":"string",
1571
+ "pattern":"^[0-9]+\\.[0-9]+\\.[0-9]+$",
1572
+ "description":"Semantic version of the contract (e.g., '2.1.0')"
1573
+ },
1574
+ "status":{
1575
+ "type":"string",
1576
+ "enum":["stable","deprecated","experimental"],
1577
+ "default":"stable",
1578
+ "description":"Contract status: stable (production-ready), deprecated (being phased out), experimental (testing)"
1579
+ },
1580
+ "compatibility":{
1581
+ "$ref":"#/$defs/compatibilityConfig",
1582
+ "description":"Compatibility information for this contract version"
1583
+ },
1584
+ "deprecation":{
1585
+ "$ref":"#/$defs/deprecationConfig",
1586
+ "description":"Deprecation information if status is 'deprecated'"
1587
+ }
1588
+ },
1589
+ "additionalProperties":false
1590
+ }
1591
+ },
1592
+ "additionalProperties":false
1593
+ }