@aifabrix/builder 2.7.0 → 2.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/.cursor/rules/project-rules.mdc +680 -0
  2. package/integration/hubspot/README.md +136 -0
  3. package/integration/hubspot/env.template +9 -0
  4. package/integration/hubspot/hubspot-deploy-company.json +200 -0
  5. package/integration/hubspot/hubspot-deploy-contact.json +228 -0
  6. package/integration/hubspot/hubspot-deploy-deal.json +248 -0
  7. package/integration/hubspot/hubspot-deploy.json +91 -0
  8. package/integration/hubspot/variables.yaml +17 -0
  9. package/lib/app-config.js +13 -2
  10. package/lib/app-deploy.js +9 -3
  11. package/lib/app-dockerfile.js +14 -1
  12. package/lib/app-prompts.js +177 -13
  13. package/lib/app-push.js +16 -1
  14. package/lib/app-register.js +37 -5
  15. package/lib/app-rotate-secret.js +10 -0
  16. package/lib/app-run.js +19 -0
  17. package/lib/app.js +70 -25
  18. package/lib/audit-logger.js +9 -4
  19. package/lib/build.js +25 -13
  20. package/lib/cli.js +109 -2
  21. package/lib/commands/login.js +40 -3
  22. package/lib/config.js +121 -114
  23. package/lib/datasource-deploy.js +14 -20
  24. package/lib/environment-deploy.js +305 -0
  25. package/lib/external-system-deploy.js +345 -0
  26. package/lib/external-system-download.js +431 -0
  27. package/lib/external-system-generator.js +190 -0
  28. package/lib/external-system-test.js +446 -0
  29. package/lib/generator-builders.js +323 -0
  30. package/lib/generator.js +200 -292
  31. package/lib/schema/application-schema.json +830 -800
  32. package/lib/schema/external-datasource.schema.json +868 -46
  33. package/lib/schema/external-system.schema.json +98 -80
  34. package/lib/schema/infrastructure-schema.json +1 -1
  35. package/lib/templates.js +32 -1
  36. package/lib/utils/cli-utils.js +4 -4
  37. package/lib/utils/device-code.js +10 -2
  38. package/lib/utils/external-system-display.js +159 -0
  39. package/lib/utils/external-system-validators.js +245 -0
  40. package/lib/utils/paths.js +151 -1
  41. package/lib/utils/schema-resolver.js +7 -2
  42. package/lib/utils/token-encryption.js +68 -0
  43. package/lib/validator.js +52 -5
  44. package/package.json +1 -1
  45. package/tatus +181 -0
  46. package/templates/external-system/external-datasource.json.hbs +55 -0
  47. package/templates/external-system/external-system.json.hbs +37 -0
package/lib/schema/external-datasource.schema.json CHANGED
@@ -1,8 +1,65 @@
1
1
  {
2
2
  "$schema":"https://json-schema.org/draft/2020-12/schema",
3
- "$id":"https://aifabrix.ai/schemas/external-datasource.schema.json",
3
+ "$id":"https://raw.githubusercontent.com/esystemsdev/aifabrix-builder/refs/heads/main/lib/schema/external-datasource.schema.json",
4
4
  "title":"External Data Source",
5
- "description":"Configuration for AI Fabrix ExternalDataSource entities. Includes metadata schema, ABAC access fields, transformation mappings, OpenAPI/MCP exposure, and sync behavior.",
5
+ "description":"Configuration for AI Fabrix ExternalDataSource entities. Includes metadata schema, ABAC access fields, transformation mappings, OpenAPI/MCP exposure, execution logic, and sync behavior.",
6
+ "metadata":{
7
+ "key":"external-datasource-schema",
8
+ "name":"External Data Source Configuration Schema",
9
+ "description":"JSON schema for validating ExternalDataSource configuration files",
10
+ "version":"1.1.0",
11
+ "type":"schema",
12
+ "category":"integration",
13
+ "author":"AI Fabrix Team",
14
+ "createdAt":"2024-01-01T00:00:00Z",
15
+ "updatedAt":"2025-01-01T00:00:00Z",
16
+ "compatibility":{
17
+ "minVersion":"1.0.0",
18
+ "maxVersion":"2.0.0",
19
+ "deprecated":false
20
+ },
21
+ "tags":[
22
+ "schema",
23
+ "external-datasource",
24
+ "dataplane",
25
+ "integration",
26
+ "validation",
27
+ "cip",
28
+ "execution"
29
+ ],
30
+ "dependencies":[
31
+ "external-system.schema.json"
32
+ ],
33
+ "changelog":[
34
+ {
35
+ "version":"1.0.0",
36
+ "date":"2024-01-01T00:00:00Z",
37
+ "changes":[
38
+ "Initial schema for External Data Sources",
39
+ "Added fieldMappings DSL with transformation expressions",
40
+ "Added metadataSchema validation",
41
+ "Added exposed fields configuration for MCP/OpenAPI",
42
+ "Added ABAC accessFields validation",
43
+ "Added OpenAPI and MCP operation configurations",
44
+ "Added sync configuration and validation rules"
45
+ ],
46
+ "breaking":false
47
+ },
48
+ {
49
+ "version":"1.1.0",
50
+ "date":"2025-01-01T00:00:00Z",
51
+ "changes":[
52
+ "Added execution.engine with CIP and Python execution modes",
53
+ "Introduced CIP definition ($defs.cipDefinition and step types)",
54
+ "Added capabilities for list/get/create/update/delete",
55
+ "Extended fieldMappings with semantic metadata for AI",
56
+ "Added exposure profiles for different AI/analytics views",
57
+ "Added data quality rules, indexing, and AI context hints"
58
+ ],
59
+ "breaking":false
60
+ }
61
+ ]
62
+ },
6
63
  "type":"object",
7
64
  "required":[
8
65
  "key",
@@ -38,6 +95,18 @@
38
95
  "type":"string",
39
96
  "pattern":"^[a-z0-9-]+$"
40
97
  },
98
+ "resourceType":{
99
+ "type":"string",
100
+ "enum":[
101
+ "customer",
102
+ "contact",
103
+ "person",
104
+ "document",
105
+ "deal"
106
+ ],
107
+ "default":"document",
108
+ "description":"Resource type for the datasource. Maps to database ResourceType enum."
109
+ },
41
110
  "version":{
42
111
  "type":"string",
43
112
  "pattern":"^[0-9]+\\.[0-9]+\\.[0-9]+$",
@@ -93,10 +162,44 @@
93
162
  ]
94
163
  },
95
164
  "description":{
96
- "type":"string"
165
+ "type":"string",
166
+ "description":"Technical description of the normalized field."
97
167
  },
98
168
  "required":{
99
169
  "type":"boolean"
170
+ },
171
+ "semantic":{
172
+ "type":"object",
173
+ "description":"Semantic metadata for AI agents and schema generation.",
174
+ "properties":{
175
+ "title":{
176
+ "type":"string",
177
+ "description":"Human-friendly label for this field."
178
+ },
179
+ "description":{
180
+ "type":"string",
181
+ "description":"Business-level description of the field and how it is used."
182
+ },
183
+ "example":{
184
+ "description":"Example value for this field.",
185
+ "type":["string","number","boolean","object","array","null"]
186
+ },
187
+ "categories":{
188
+ "type":"array",
189
+ "description":"Logical categories/tags (e.g. ['sales', 'revenue']).",
190
+ "items":{
191
+ "type":"string"
192
+ }
193
+ },
194
+ "synonyms":{
195
+ "type":"array",
196
+ "description":"Optional list of synonyms used in natural language (e.g. 'opportunity', 'sales-case').",
197
+ "items":{
198
+ "type":"string"
199
+ }
200
+ }
201
+ },
202
+ "additionalProperties":false
100
203
  }
101
204
  }
102
205
  }
@@ -151,6 +254,19 @@
151
254
  "description":{
152
255
  "type":"string",
153
256
  "description":"Human-readable description of the exposure model for this data source."
257
+ },
258
+ "profiles":{
259
+ "type":"object",
260
+ "description":"Named exposure profiles mapping profile keys to lists of normalized field names. Useful for AI-light, AI-embedding, analytics, etc.",
261
+ "additionalProperties":{
262
+ "type":"array",
263
+ "items":{
264
+ "type":"string",
265
+ "pattern":"^[a-zA-Z0-9_]+$"
266
+ },
267
+ "minItems":1,
268
+ "uniqueItems":true
269
+ }
154
270
  }
155
271
  },
156
272
  "required":[
@@ -190,7 +306,7 @@
190
306
  },
191
307
  "openapi":{
192
308
  "type":"object",
193
- "description":"OpenAPI-driven connector configuration.",
309
+ "description":"OpenAPI-driven connector configuration. Only includes endpoints selected during wizard onboarding. To add more endpoints, run the wizard again.",
194
310
  "properties":{
195
311
  "enabled":{
196
312
  "type":"boolean",
@@ -208,6 +324,7 @@
208
324
  },
209
325
  "operations":{
210
326
  "type":"object",
327
+ "description":"Selected operations from the OpenAPI document. Only endpoints chosen during wizard onboarding are included.",
211
328
  "properties":{
212
329
  "list":{
213
330
  "type":"object",
@@ -313,48 +430,6 @@
313
430
  }
314
431
  }
315
432
  },
316
- "mcp":{
317
- "type":"object",
318
- "description":"Model Context Protocol exposure.",
319
- "properties":{
320
- "enabled":{
321
- "type":"boolean",
322
- "default":false
323
- },
324
- "toolName":{
325
- "type":"string",
326
- "pattern":"^[a-z0-9-]+$"
327
- },
328
- "resourceName":{
329
- "type":"string",
330
- "pattern":"^[a-z0-9-:]+$"
331
- },
332
- "operations":{
333
- "type":"array",
334
- "items":{
335
- "type":"object",
336
- "required":[
337
- "name",
338
- "permission"
339
- ],
340
- "properties":{
341
- "name":{
342
- "type":"string",
343
- "pattern":"^[a-z0-9-]+$"
344
- },
345
- "permission":{
346
- "type":"string",
347
- "description":"Must match securityModel.permissions",
348
- "pattern":"^[a-z0-9-:]+$"
349
- },
350
- "description":{
351
- "type":"string"
352
- }
353
- }
354
- }
355
- }
356
- }
357
- },
358
433
  "validation":{
359
434
  "type":"object",
360
435
  "description":"Advanced validation rules applied before saving normalized metadata.",
@@ -399,6 +474,252 @@
399
474
  }
400
475
  }
401
476
  },
477
+ "quality":{
478
+ "type":"object",
479
+ "description":"Data quality rules applied after mapping but before persistence and AI exposure.",
480
+ "properties":{
481
+ "rejectIf":{
482
+ "type":"array",
483
+ "description":"List of conditions that cause a record to be rejected.",
484
+ "items":{
485
+ "type":"object",
486
+ "required":[
487
+ "field",
488
+ "operator"
489
+ ],
490
+ "properties":{
491
+ "field":{
492
+ "type":"string",
493
+ "pattern":"^[a-zA-Z0-9_]+$",
494
+ "description":"Normalized field name to evaluate."
495
+ },
496
+ "operator":{
497
+ "type":"string",
498
+ "enum":[
499
+ "missing",
500
+ "null",
501
+ "empty",
502
+ "invalidRef",
503
+ "lessThan",
504
+ "greaterThan",
505
+ "equal",
506
+ "notEqual"
507
+ ]
508
+ },
509
+ "value":{
510
+ "description":"Comparison value for applicable operators.",
511
+ "type":["string","number","boolean","null"]
512
+ },
513
+ "message":{
514
+ "type":"string",
515
+ "description":"Optional human-readable reason for rejection."
516
+ }
517
+ },
518
+ "additionalProperties":false
519
+ }
520
+ }
521
+ },
522
+ "additionalProperties":false
523
+ },
524
+ "indexing":{
525
+ "type":"object",
526
+ "description":"Indexing and embedding strategy for this datasource.",
527
+ "properties":{
528
+ "embedding":{
529
+ "type":"array",
530
+ "description":"List of normalized fields whose values should be concatenated and embedded for vector search.",
531
+ "items":{
532
+ "type":"string",
533
+ "pattern":"^[a-zA-Z0-9_]+$"
534
+ },
535
+ "minItems":1,
536
+ "uniqueItems":true
537
+ },
538
+ "uniqueKey":{
539
+ "type":"string",
540
+ "pattern":"^[a-zA-Z0-9_]+$",
541
+ "description":"Normalized field used as unique identifier for deduplication and upserts."
542
+ },
543
+ "dedupeStrategy":{
544
+ "type":"string",
545
+ "enum":[
546
+ "reject",
547
+ "latest-wins",
548
+ "first-wins",
549
+ "merge"
550
+ ],
551
+ "default":"latest-wins",
552
+ "description":"Strategy to apply when multiple records share the same uniqueKey."
553
+ }
554
+ },
555
+ "additionalProperties":false
556
+ },
557
+ "context":{
558
+ "type":"object",
559
+ "description":"Natural-language and semantic hints for AI agents.",
560
+ "properties":{
561
+ "semanticTags":{
562
+ "type":"array",
563
+ "description":"High-level tags describing this datasource (e.g. ['crm', 'customer-data', 'deal']).",
564
+ "items":{
565
+ "type":"string"
566
+ }
567
+ },
568
+ "synonyms":{
569
+ "type":"array",
570
+ "description":"Terms users might use in natural language to refer to this datasource or its primary entity.",
571
+ "items":{
572
+ "type":"string"
573
+ }
574
+ },
575
+ "naturalLanguageHints":{
576
+ "type":"array",
577
+ "description":"Free-form hints for LLMs on how to interpret or use this datasource.",
578
+ "items":{
579
+ "type":"string"
580
+ }
581
+ }
582
+ },
583
+ "additionalProperties":false
584
+ },
585
+ "documentStorage":{
586
+ "type":"object",
587
+ "description":"Document storage configuration (optional, enables vector storage)",
588
+ "properties":{
589
+ "enabled":{
590
+ "type":"boolean",
591
+ "default":true
592
+ },
593
+ "flowiseInstanceId":{
594
+ "type":"string"
595
+ },
596
+ "twoPhaseSync":{
597
+ "type":"boolean",
598
+ "default":true
599
+ },
600
+ "binaryOperation":{
601
+ "type":"object",
602
+ "description":"Configuration for binary document retrieval. Supports CIP operations or direct HTTP/OpenAPI.",
603
+ "oneOf":[
604
+ {
605
+ "required":[
606
+ "cipOperationRef",
607
+ "responseType"
608
+ ],
609
+ "properties":{
610
+ "cipOperationRef":{
611
+ "type":"string",
612
+ "description":"Reference to CIP operation (e.g., 'get' or custom operation name)"
613
+ },
614
+ "responseType":{
615
+ "type":"string",
616
+ "enum":[
617
+ "binary",
618
+ "base64",
619
+ "json"
620
+ ],
621
+ "default":"binary"
622
+ },
623
+ "binaryField":{
624
+ "type":"string"
625
+ },
626
+ "parameterMapping":{
627
+ "type":"object",
628
+ "additionalProperties":{
629
+ "type":"string"
630
+ }
631
+ }
632
+ },
633
+ "additionalProperties":false
634
+ },
635
+ {
636
+ "required":[
637
+ "path",
638
+ "responseType"
639
+ ],
640
+ "properties":{
641
+ "operationId":{
642
+ "type":"string"
643
+ },
644
+ "method":{
645
+ "type":"string",
646
+ "enum":[
647
+ "GET",
648
+ "POST",
649
+ "PUT"
650
+ ],
651
+ "default":"GET"
652
+ },
653
+ "path":{
654
+ "type":"string"
655
+ },
656
+ "queryParameters":{
657
+ "type":"object",
658
+ "additionalProperties":{
659
+ "type":"string"
660
+ }
661
+ },
662
+ "headers":{
663
+ "type":"object",
664
+ "additionalProperties":{
665
+ "type":"string"
666
+ }
667
+ },
668
+ "requestBody":{
669
+ "type":"object"
670
+ },
671
+ "responseType":{
672
+ "type":"string",
673
+ "enum":[
674
+ "binary",
675
+ "base64",
676
+ "json"
677
+ ],
678
+ "default":"binary"
679
+ },
680
+ "binaryField":{
681
+ "type":"string"
682
+ },
683
+ "parameterMapping":{
684
+ "type":"object",
685
+ "additionalProperties":{
686
+ "type":"string"
687
+ }
688
+ }
689
+ },
690
+ "additionalProperties":false
691
+ }
692
+ ]
693
+ },
694
+ "processing":{
695
+ "type":"object",
696
+ "properties":{
697
+ "fileStoragePath":{
698
+ "type":"string",
699
+ "default":"/data/documents"
700
+ },
701
+ "aiValidation":{
702
+ "type":"object"
703
+ },
704
+ "spacyEnrichment":{
705
+ "type":"object"
706
+ },
707
+ "notifications":{
708
+ "type":"object"
709
+ }
710
+ },
711
+ "additionalProperties":false
712
+ },
713
+ "credentialId":{
714
+ "type":"string"
715
+ }
716
+ },
717
+ "required":[
718
+ "flowiseInstanceId",
719
+ "binaryOperation"
720
+ ],
721
+ "additionalProperties":false
722
+ },
402
723
  "portalInput":{
403
724
  "type":"array",
404
725
  "description":"Optional UI metadata definition for the AI Fabrix portal.",
@@ -458,6 +779,507 @@
458
779
  }
459
780
  }
460
781
  }
782
+ },
783
+ "testPayload":{
784
+ "type":"object",
785
+ "description":"Test payload configuration for unit and integration testing",
786
+ "properties":{
787
+ "payloadTemplate":{
788
+ "type":"object",
789
+ "description":"Sample payload matching the expected API response structure. Used for testing field mappings and metadata schema validation.",
790
+ "additionalProperties":true
791
+ },
792
+ "expectedResult":{
793
+ "type":"object",
794
+ "description":"Expected normalized result after field mapping transformations (optional, for validation)",
795
+ "additionalProperties":true
796
+ }
797
+ },
798
+ "additionalProperties":false
799
+ },
800
+ "capabilities":{
801
+ "type":"object",
802
+ "description":"Declares which logical operations are supported by this datasource.",
803
+ "properties":{
804
+ "list":{
805
+ "type":"boolean",
806
+ "default":true
807
+ },
808
+ "get":{
809
+ "type":"boolean",
810
+ "default":false
811
+ },
812
+ "create":{
813
+ "type":"boolean",
814
+ "default":false
815
+ },
816
+ "update":{
817
+ "type":"boolean",
818
+ "default":false
819
+ },
820
+ "delete":{
821
+ "type":"boolean",
822
+ "default":false
823
+ }
824
+ },
825
+ "additionalProperties":false
826
+ },
827
+ "execution":{
828
+ "type":"object",
829
+ "description":"Execution engine configuration for this datasource (CIP or Python). CIP is the native declarative mode.",
830
+ "required":[
831
+ "engine"
832
+ ],
833
+ "properties":{
834
+ "engine":{
835
+ "type":"string",
836
+ "enum":[
837
+ "cip",
838
+ "python"
839
+ ],
840
+ "description":"Execution engine. 'cip' for declarative pipelines, 'python' for custom handlers."
841
+ },
842
+ "cip":{
843
+ "$ref":"#/$defs/cipDefinition"
844
+ },
845
+ "python":{
846
+ "$ref":"#/$defs/pythonExecution"
847
+ }
848
+ },
849
+ "additionalProperties":false
850
+ }
851
+ },
852
+ "$defs":{
853
+ "pythonExecution":{
854
+ "type":"object",
855
+ "description":"Python-based execution config for advanced/custom use cases.",
856
+ "properties":{
857
+ "entrypoint":{
858
+ "type":"string",
859
+ "description":"Python callable in module notation, e.g. 'handlers.hubspot_deal.list_records'."
860
+ },
861
+ "codeRef":{
862
+ "type":"string",
863
+ "description":"Optional reference to external code bundle identifier or embedded block key."
864
+ }
865
+ },
866
+ "required":[
867
+ "entrypoint"
868
+ ],
869
+ "additionalProperties":false
870
+ },
871
+ "cipLineage":{
872
+ "type":"object",
873
+ "description":"Optional lineage metadata for a CIP operation or step.",
874
+ "properties":{
875
+ "producer":{
876
+ "type":"string",
877
+ "description":"Upstream system or connector name (e.g. 'hubspot')."
878
+ },
879
+ "entity":{
880
+ "type":"string",
881
+ "description":"Upstream entity or resource (e.g. 'deal')."
882
+ },
883
+ "operation":{
884
+ "type":"string",
885
+ "description":"Logical operation (e.g. 'list', 'get', 'sync-job-1')."
886
+ }
887
+ },
888
+ "additionalProperties":false
889
+ },
890
+ "cipDefinition":{
891
+ "type":"object",
892
+ "description":"Composable Integration Pipeline (CIP) definition for this datasource.",
893
+ "required":[
894
+ "version",
895
+ "operations"
896
+ ],
897
+ "properties":{
898
+ "version":{
899
+ "type":"string",
900
+ "enum":[
901
+ "1.0"
902
+ ]
903
+ },
904
+ "inputType":{
905
+ "type":"string",
906
+ "const":"void",
907
+ "description":"Logical input type of the pipeline. Reserved for future extensions."
908
+ },
909
+ "outputType":{
910
+ "type":"string",
911
+ "const":"records",
912
+ "description":"Logical output type of the pipeline."
913
+ },
914
+ "operations":{
915
+ "type":"object",
916
+ "description":"Logical operations (list/get/create/update/delete) implemented for this datasource.",
917
+ "properties":{
918
+ "list":{
919
+ "$ref":"#/$defs/cipOperation"
920
+ },
921
+ "get":{
922
+ "$ref":"#/$defs/cipOperation"
923
+ },
924
+ "create":{
925
+ "$ref":"#/$defs/cipOperation"
926
+ },
927
+ "update":{
928
+ "$ref":"#/$defs/cipOperation"
929
+ },
930
+ "delete":{
931
+ "$ref":"#/$defs/cipOperation"
932
+ }
933
+ },
934
+ "additionalProperties":false
935
+ }
936
+ },
937
+ "additionalProperties":false
938
+ },
939
+ "cipOperation":{
940
+ "type":"object",
941
+ "required":[
942
+ "steps"
943
+ ],
944
+ "properties":{
945
+ "description":{
946
+ "type":"string"
947
+ },
948
+ "enabled":{
949
+ "type":"boolean",
950
+ "default":true
951
+ },
952
+ "lineage":{
953
+ "$ref":"#/$defs/cipLineage"
954
+ },
955
+ "steps":{
956
+ "type":"array",
957
+ "minItems":1,
958
+ "items":{
959
+ "$ref":"#/$defs/cipStep"
960
+ }
961
+ }
962
+ },
963
+ "additionalProperties":false
964
+ },
965
+ "cipStep":{
966
+ "description":"One step in the CIP pipeline.",
967
+ "oneOf":[
968
+ {
969
+ "$ref":"#/$defs/cipStepFetch"
970
+ },
971
+ {
972
+ "$ref":"#/$defs/cipStepPaginate"
973
+ },
974
+ {
975
+ "$ref":"#/$defs/cipStepMap"
976
+ },
977
+ {
978
+ "$ref":"#/$defs/cipStepFilter"
979
+ },
980
+ {
981
+ "$ref":"#/$defs/cipStepOutput"
982
+ },
983
+ {
984
+ "$ref":"#/$defs/cipStepPythonInline"
985
+ }
986
+ ]
987
+ },
988
+ "cipStepFetch":{
989
+ "type":"object",
990
+ "required":[
991
+ "fetch"
992
+ ],
993
+ "properties":{
994
+ "fetch":{
995
+ "type":"object",
996
+ "description":"Fetch raw records from an upstream system.",
997
+ "required":[
998
+ "source"
999
+ ],
1000
+ "properties":{
1001
+ "source":{
1002
+ "type":"string",
1003
+ "enum":[
1004
+ "openapi",
1005
+ "http"
1006
+ ],
1007
+ "description":"If 'openapi', refer to openapi.operations.*. If 'http', use explicit method/path."
1008
+ },
1009
+ "openapiRef":{
1010
+ "type":"string",
1011
+ "enum":[
1012
+ "list",
1013
+ "get",
1014
+ "create",
1015
+ "update",
1016
+ "delete"
1017
+ ],
1018
+ "description":"Reference to openapi.operations.<openapiRef> entry."
1019
+ },
1020
+ "operationId":{
1021
+ "type":"string",
1022
+ "description":"OpenAPI operationId to call when source='openapi'. Optional when openapiRef is used."
1023
+ },
1024
+ "method":{
1025
+ "type":"string",
1026
+ "enum":[
1027
+ "GET",
1028
+ "POST"
1029
+ ],
1030
+ "description":"Explicit HTTP method when source='http'."
1031
+ },
1032
+ "path":{
1033
+ "type":"string",
1034
+ "description":"Explicit HTTP path when source='http'."
1035
+ },
1036
+ "query":{
1037
+ "type":"object",
1038
+ "description":"Static query parameters. ABAC/runtime can still override or append at runtime.",
1039
+ "additionalProperties":{
1040
+ "type":[
1041
+ "string",
1042
+ "number",
1043
+ "boolean",
1044
+ "null"
1045
+ ]
1046
+ }
1047
+ },
1048
+ "bodyTemplate":{
1049
+ "description":"Optional static request body or template for POST/PUT fetches.",
1050
+ "type":["object","string","null"]
1051
+ }
1052
+ },
1053
+ "allOf":[
1054
+ {
1055
+ "if":{
1056
+ "properties":{
1057
+ "source":{
1058
+ "const":"openapi"
1059
+ }
1060
+ }
1061
+ },
1062
+ "then":{
1063
+ "anyOf":[
1064
+ {
1065
+ "required":[
1066
+ "openapiRef"
1067
+ ]
1068
+ },
1069
+ {
1070
+ "required":[
1071
+ "operationId"
1072
+ ]
1073
+ }
1074
+ ]
1075
+ }
1076
+ },
1077
+ {
1078
+ "if":{
1079
+ "properties":{
1080
+ "source":{
1081
+ "const":"http"
1082
+ }
1083
+ }
1084
+ },
1085
+ "then":{
1086
+ "required":[
1087
+ "method",
1088
+ "path"
1089
+ ]
1090
+ }
1091
+ }
1092
+ ],
1093
+ "additionalProperties":false
1094
+ },
1095
+ "lineage":{
1096
+ "$ref":"#/$defs/cipLineage"
1097
+ }
1098
+ },
1099
+ "additionalProperties":false
1100
+ },
1101
+ "cipStepPaginate":{
1102
+ "type":"object",
1103
+ "required":[
1104
+ "paginate"
1105
+ ],
1106
+ "properties":{
1107
+ "paginate":{
1108
+ "type":"object",
1109
+ "description":"Pagination strategy applied to results from the preceding fetch.",
1110
+ "required":[
1111
+ "strategy"
1112
+ ],
1113
+ "properties":{
1114
+ "strategy":{
1115
+ "type":"string",
1116
+ "enum":[
1117
+ "cursor",
1118
+ "page",
1119
+ "offset",
1120
+ "none"
1121
+ ]
1122
+ },
1123
+ "cursorField":{
1124
+ "type":"string",
1125
+ "description":"Field in the response payload containing the cursor value (for 'cursor' strategy)."
1126
+ },
1127
+ "cursorParam":{
1128
+ "type":"string",
1129
+ "description":"Name of the query parameter used to send the cursor."
1130
+ },
1131
+ "pageParam":{
1132
+ "type":"string",
1133
+ "description":"Name of the query parameter used to send the page number (for 'page' strategy)."
1134
+ },
1135
+ "pageSizeParam":{
1136
+ "type":"string",
1137
+ "description":"Name of the query parameter used to send the page size."
1138
+ },
1139
+ "pageSize":{
1140
+ "type":"integer",
1141
+ "minimum":1,
1142
+ "description":"Default page size."
1143
+ },
1144
+ "maxPages":{
1145
+ "type":"integer",
1146
+ "minimum":1,
1147
+ "description":"Maximum number of pages to fetch."
1148
+ }
1149
+ },
1150
+ "additionalProperties":false
1151
+ },
1152
+ "lineage":{
1153
+ "$ref":"#/$defs/cipLineage"
1154
+ }
1155
+ },
1156
+ "additionalProperties":false
1157
+ },
1158
+ "cipStepMap":{
1159
+ "type":"object",
1160
+ "required":[
1161
+ "map"
1162
+ ],
1163
+ "properties":{
1164
+ "map":{
1165
+ "type":"object",
1166
+ "description":"Transform raw API records into the normalized metadata model.",
1167
+ "properties":{
1168
+ "inputPath":{
1169
+ "type":"string",
1170
+ "default":"$.items[*]",
1171
+ "description":"JSONPath-like expression to iterate over records in the upstream payload."
1172
+ },
1173
+ "useFieldMappings":{
1174
+ "type":"boolean",
1175
+ "default":true,
1176
+ "description":"If true, apply the datasource.fieldMappings.fields expressions; no inline mapping needed."
1177
+ },
1178
+ "inline":{
1179
+ "type":"object",
1180
+ "description":"Optional inline mapping overrides { normalizedField -> expression }.",
1181
+ "additionalProperties":{
1182
+ "type":"string",
1183
+ "description":"Pipe-based DSL expression overriding fieldMappings expression."
1184
+ }
1185
+ }
1186
+ },
1187
+ "additionalProperties":false
1188
+ },
1189
+ "lineage":{
1190
+ "$ref":"#/$defs/cipLineage"
1191
+ }
1192
+ },
1193
+ "additionalProperties":false
1194
+ },
1195
+ "cipStepFilter":{
1196
+ "type":"object",
1197
+ "required":[
1198
+ "filter"
1199
+ ],
1200
+ "properties":{
1201
+ "filter":{
1202
+ "type":"object",
1203
+ "description":"Filter mapped records before output. ABAC is applied automatically based on fieldMappings.accessFields.",
1204
+ "properties":{
1205
+ "enforceAbac":{
1206
+ "type":"boolean",
1207
+ "default":true
1208
+ },
1209
+ "expression":{
1210
+ "type":"string",
1211
+ "description":"Optional additional filter expression (e.g. '@.status == \"active\"')."
1212
+ }
1213
+ },
1214
+ "additionalProperties":false
1215
+ },
1216
+ "lineage":{
1217
+ "$ref":"#/$defs/cipLineage"
1218
+ }
1219
+ },
1220
+ "additionalProperties":false
1221
+ },
1222
+ "cipStepOutput":{
1223
+ "type":"object",
1224
+ "required":[
1225
+ "output"
1226
+ ],
1227
+ "properties":{
1228
+ "output":{
1229
+ "type":"object",
1230
+ "description":"Finalize the CIP pipeline and emit records.",
1231
+ "properties":{
1232
+ "mode":{
1233
+ "type":"string",
1234
+ "enum":[
1235
+ "records"
1236
+ ],
1237
+ "default":"records"
1238
+ },
1239
+ "limit":{
1240
+ "type":"integer",
1241
+ "minimum":1,
1242
+ "description":"Optional soft limit for records returned by list operation; sync runner can override."
1243
+ },
1244
+ "includeConfidence":{
1245
+ "type":"boolean",
1246
+ "default":false,
1247
+ "description":"If true, the pipeline will compute a confidence score per record based on quality checks, mapping completeness, and source hints."
1248
+ }
1249
+ },
1250
+ "additionalProperties":false
1251
+ },
1252
+ "lineage":{
1253
+ "$ref":"#/$defs/cipLineage"
1254
+ }
1255
+ },
1256
+ "additionalProperties":false
1257
+ },
1258
+ "cipStepPythonInline":{
1259
+ "type":"object",
1260
+ "required":[
1261
+ "pythonInline"
1262
+ ],
1263
+ "properties":{
1264
+ "pythonInline":{
1265
+ "type":"object",
1266
+ "description":"Sandboxed inline Python transformation, operating on the current record batch.",
1267
+ "properties":{
1268
+ "code":{
1269
+ "type":"string",
1270
+ "description":"Small, pure function snippet; no imports, no IO, no network. Runtime must enforce sandboxing."
1271
+ }
1272
+ },
1273
+ "required":[
1274
+ "code"
1275
+ ],
1276
+ "additionalProperties":false
1277
+ },
1278
+ "lineage":{
1279
+ "$ref":"#/$defs/cipLineage"
1280
+ }
1281
+ },
1282
+ "additionalProperties":false
461
1283
  }
462
1284
  },
463
1285
  "additionalProperties":false