@objectstack/spec 0.3.3 → 0.4.2

package/json-schema/automation/DataSyncConfig.json

@@ -0,0 +1,355 @@
+{
+  "$ref": "#/definitions/DataSyncConfig",
+  "definitions": {
+    "DataSyncConfig": {
+      "type": "object",
+      "properties": {
+        "name": {
+          "type": "string",
+          "pattern": "^[a-z_][a-z0-9_]*$",
+          "description": "Sync configuration name (snake_case)"
+        },
+        "label": {
+          "type": "string",
+          "description": "Sync display name"
+        },
+        "description": {
+          "type": "string",
+          "description": "Sync description"
+        },
+        "source": {
+          "type": "object",
+          "properties": {
+            "object": {
+              "type": "string",
+              "description": "ObjectStack object name"
+            },
+            "filters": {
+              "description": "Filter conditions"
+            },
+            "fields": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "description": "Fields to sync"
+            },
+            "connectorInstanceId": {
+              "type": "string",
+              "description": "Connector instance ID"
+            },
+            "externalResource": {
+              "type": "string",
+              "description": "External resource ID"
+            }
+          },
+          "additionalProperties": false,
+          "description": "Data source"
+        },
+        "destination": {
+          "type": "object",
+          "properties": {
+            "object": {
+              "type": "string",
+              "description": "ObjectStack object name"
+            },
+            "connectorInstanceId": {
+              "type": "string",
+              "description": "Connector instance ID"
+            },
+            "operation": {
+              "type": "string",
+              "enum": [
+                "insert",
+                "update",
+                "upsert",
+                "delete",
+                "sync"
+              ],
+              "description": "Sync operation"
+            },
+            "mapping": {
+              "anyOf": [
+                {
+                  "type": "object",
+                  "additionalProperties": {
+                    "type": "string"
+                  }
+                },
+                {
+                  "type": "array",
+                  "items": {
+                    "type": "object",
+                    "properties": {
+                      "sourceField": {
+                        "type": "string",
+                        "description": "Source field name"
+                      },
+                      "destinationField": {
+                        "type": "string",
+                        "description": "Destination field name"
+                      },
+                      "transform": {
+                        "type": "string",
+                        "description": "Transformation formula"
+                      },
+                      "default": {
+                        "description": "Default value"
+                      },
+                      "syncNull": {
+                        "type": "boolean",
+                        "default": false,
+                        "description": "Sync null values"
+                      }
+                    },
+                    "required": [
+                      "sourceField",
+                      "destinationField"
+                    ],
+                    "additionalProperties": false
+                  }
+                }
+              ],
+              "description": "Field mappings"
+            },
+            "externalResource": {
+              "type": "string",
+              "description": "External resource ID"
+            },
+            "matchKey": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "description": "Match key fields"
+            }
+          },
+          "required": [
+            "operation"
+          ],
+          "additionalProperties": false,
+          "description": "Data destination"
+        },
+        "direction": {
+          "type": "string",
+          "enum": [
+            "push",
+            "pull",
+            "bidirectional"
+          ],
+          "default": "push",
+          "description": "Sync direction"
+        },
+        "syncMode": {
+          "type": "string",
+          "enum": [
+            "full",
+            "incremental",
+            "realtime"
+          ],
+          "default": "incremental",
+          "description": "Sync mode"
+        },
+        "conflictResolution": {
+          "type": "string",
+          "enum": [
+            "source_wins",
+            "destination_wins",
+            "latest_wins",
+            "manual",
+            "merge"
+          ],
+          "default": "latest_wins",
+          "description": "Conflict resolution"
+        },
+        "schedule": {
+          "type": "string",
+          "description": "Cron schedule"
+        },
+        "enabled": {
+          "type": "boolean",
+          "default": true,
+          "description": "Sync enabled"
+        },
+        "changeTrackingField": {
+          "type": "string",
+          "description": "Field for change tracking"
+        },
+        "batchSize": {
+          "type": "integer",
+          "minimum": 1,
+          "maximum": 10000,
+          "default": 100,
+          "description": "Batch size for processing"
+        },
+        "retry": {
+          "type": "object",
+          "properties": {
+            "maxAttempts": {
+              "type": "integer",
+              "minimum": 0,
+              "default": 3,
+              "description": "Max retries"
+            },
+            "backoffMs": {
+              "type": "integer",
+              "minimum": 0,
+              "default": 30000,
+              "description": "Backoff duration"
+            }
+          },
+          "additionalProperties": false,
+          "description": "Retry configuration"
+        },
+        "validation": {
+          "type": "object",
+          "properties": {
+            "required": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "description": "Required fields"
+            },
+            "unique": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "description": "Unique constraint fields"
+            },
+            "custom": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "properties": {
+                  "name": {
+                    "type": "string"
+                  },
+                  "condition": {
+                    "type": "string",
+                    "description": "Validation condition"
+                  },
+                  "message": {
+                    "type": "string",
+                    "description": "Error message"
+                  }
+                },
+                "required": [
+                  "name",
+                  "condition",
+                  "message"
+                ],
+                "additionalProperties": false
+              },
+              "description": "Custom validation rules"
+            }
+          },
+          "additionalProperties": false,
+          "description": "Validation rules"
+        },
+        "errorHandling": {
+          "type": "object",
+          "properties": {
+            "onValidationError": {
+              "type": "string",
+              "enum": [
+                "skip",
+                "fail",
+                "log"
+              ],
+              "default": "skip"
+            },
+            "onSyncError": {
+              "type": "string",
+              "enum": [
+                "skip",
+                "fail",
+                "retry"
+              ],
+              "default": "retry"
+            },
+            "notifyOnError": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "description": "Email notifications"
+            }
+          },
+          "additionalProperties": false,
+          "description": "Error handling"
+        },
+        "optimization": {
+          "type": "object",
+          "properties": {
+            "parallelBatches": {
+              "type": "boolean",
+              "default": false,
+              "description": "Process batches in parallel"
+            },
+            "cacheEnabled": {
+              "type": "boolean",
+              "default": true,
+              "description": "Enable caching"
+            },
+            "compressionEnabled": {
+              "type": "boolean",
+              "default": false,
+              "description": "Enable compression"
+            }
+          },
+          "additionalProperties": false,
+          "description": "Performance optimization"
+        },
+        "audit": {
+          "type": "object",
+          "properties": {
+            "logLevel": {
+              "type": "string",
+              "enum": [
+                "none",
+                "error",
+                "warn",
+                "info",
+                "debug"
+              ],
+              "default": "info"
+            },
+            "retainLogsForDays": {
+              "type": "integer",
+              "minimum": 1,
+              "default": 30
+            },
+            "trackChanges": {
+              "type": "boolean",
+              "default": true,
+              "description": "Track all changes"
+            }
+          },
+          "additionalProperties": false,
+          "description": "Audit configuration"
+        },
+        "tags": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "description": "Sync tags"
+        },
+        "metadata": {
+          "type": "object",
+          "additionalProperties": {},
+          "description": "Custom metadata"
+        }
+      },
+      "required": [
+        "name",
+        "source",
+        "destination"
+      ],
+      "additionalProperties": false
+    }
+  },
+  "$schema": "http://json-schema.org/draft-07/schema#"
+}

package/json-schema/automation/ETLDestination.json

@@ -0,0 +1,57 @@
+{
+  "$ref": "#/definitions/ETLDestination",
+  "definitions": {
+    "ETLDestination": {
+      "type": "object",
+      "properties": {
+        "type": {
+          "type": "string",
+          "enum": [
+            "database",
+            "api",
+            "file",
+            "stream",
+            "object",
+            "warehouse",
+            "storage",
+            "spreadsheet"
+          ],
+          "description": "Destination type"
+        },
+        "connector": {
+          "type": "string",
+          "description": "Connector ID"
+        },
+        "config": {
+          "type": "object",
+          "additionalProperties": {},
+          "description": "Destination configuration"
+        },
+        "writeMode": {
+          "type": "string",
+          "enum": [
+            "append",
+            "overwrite",
+            "upsert",
+            "merge"
+          ],
+          "default": "append",
+          "description": "How to write data"
+        },
+        "primaryKey": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "description": "Primary key fields"
+        }
+      },
+      "required": [
+        "type",
+        "config"
+      ],
+      "additionalProperties": false
+    }
+  },
+  "$schema": "http://json-schema.org/draft-07/schema#"
+}

package/json-schema/automation/ETLEndpointType.json

@@ -0,0 +1,19 @@
+{
+  "$ref": "#/definitions/ETLEndpointType",
+  "definitions": {
+    "ETLEndpointType": {
+      "type": "string",
+      "enum": [
+        "database",
+        "api",
+        "file",
+        "stream",
+        "object",
+        "warehouse",
+        "storage",
+        "spreadsheet"
+      ]
+    }
+  },
+  "$schema": "http://json-schema.org/draft-07/schema#"
+}

package/json-schema/automation/ETLPipeline.json

@@ -0,0 +1,252 @@
+{
+  "$ref": "#/definitions/ETLPipeline",
+  "definitions": {
+    "ETLPipeline": {
+      "type": "object",
+      "properties": {
+        "name": {
+          "type": "string",
+          "pattern": "^[a-z_][a-z0-9_]*$",
+          "description": "Pipeline identifier (snake_case)"
+        },
+        "label": {
+          "type": "string",
+          "description": "Pipeline display name"
+        },
+        "description": {
+          "type": "string",
+          "description": "Pipeline description"
+        },
+        "source": {
+          "type": "object",
+          "properties": {
+            "type": {
+              "type": "string",
+              "enum": [
+                "database",
+                "api",
+                "file",
+                "stream",
+                "object",
+                "warehouse",
+                "storage",
+                "spreadsheet"
+              ],
+              "description": "Source type"
+            },
+            "connector": {
+              "type": "string",
+              "description": "Connector ID"
+            },
+            "config": {
+              "type": "object",
+              "additionalProperties": {},
+              "description": "Source configuration"
+            },
+            "incremental": {
+              "type": "object",
+              "properties": {
+                "enabled": {
+                  "type": "boolean",
+                  "default": false
+                },
+                "cursorField": {
+                  "type": "string",
+                  "description": "Field to track progress (e.g., updated_at)"
+                },
+                "cursorValue": {
+                  "description": "Last processed value"
+                }
+              },
+              "required": [
+                "cursorField"
+              ],
+              "additionalProperties": false,
+              "description": "Incremental extraction config"
+            }
+          },
+          "required": [
+            "type",
+            "config"
+          ],
+          "additionalProperties": false,
+          "description": "Data source"
+        },
+        "destination": {
+          "type": "object",
+          "properties": {
+            "type": {
+              "type": "string",
+              "enum": [
+                "database",
+                "api",
+                "file",
+                "stream",
+                "object",
+                "warehouse",
+                "storage",
+                "spreadsheet"
+              ],
+              "description": "Destination type"
+            },
+            "connector": {
+              "type": "string",
+              "description": "Connector ID"
+            },
+            "config": {
+              "type": "object",
+              "additionalProperties": {},
+              "description": "Destination configuration"
+            },
+            "writeMode": {
+              "type": "string",
+              "enum": [
+                "append",
+                "overwrite",
+                "upsert",
+                "merge"
+              ],
+              "default": "append",
+              "description": "How to write data"
+            },
+            "primaryKey": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "description": "Primary key fields"
+            }
+          },
+          "required": [
+            "type",
+            "config"
+          ],
+          "additionalProperties": false,
+          "description": "Data destination"
+        },
+        "transformations": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "name": {
+                "type": "string",
+                "description": "Transformation name"
+              },
+              "type": {
+                "type": "string",
+                "enum": [
+                  "map",
+                  "filter",
+                  "aggregate",
+                  "join",
+                  "script",
+                  "lookup",
+                  "split",
+                  "merge",
+                  "normalize",
+                  "deduplicate"
+                ],
+                "description": "Transformation type"
+              },
+              "config": {
+                "type": "object",
+                "additionalProperties": {},
+                "description": "Transformation config"
+              },
+              "continueOnError": {
+                "type": "boolean",
+                "default": false,
+                "description": "Continue on error"
+              }
+            },
+            "required": [
+              "type",
+              "config"
+            ],
+            "additionalProperties": false
+          },
+          "description": "Transformation pipeline"
+        },
+        "syncMode": {
+          "type": "string",
+          "enum": [
+            "full",
+            "incremental",
+            "cdc"
+          ],
+          "default": "full",
+          "description": "Sync mode"
+        },
+        "schedule": {
+          "type": "string",
+          "description": "Cron schedule expression"
+        },
+        "enabled": {
+          "type": "boolean",
+          "default": true,
+          "description": "Pipeline enabled status"
+        },
+        "retry": {
+          "type": "object",
+          "properties": {
+            "maxAttempts": {
+              "type": "integer",
+              "minimum": 0,
+              "default": 3,
+              "description": "Max retry attempts"
+            },
+            "backoffMs": {
+              "type": "integer",
+              "minimum": 0,
+              "default": 60000,
+              "description": "Backoff in milliseconds"
+            }
+          },
+          "additionalProperties": false,
+          "description": "Retry configuration"
+        },
+        "notifications": {
+          "type": "object",
+          "properties": {
+            "onSuccess": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "description": "Email addresses for success notifications"
+            },
+            "onFailure": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "description": "Email addresses for failure notifications"
+            }
+          },
+          "additionalProperties": false,
+          "description": "Notification settings"
+        },
+        "tags": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "description": "Pipeline tags"
+        },
+        "metadata": {
+          "type": "object",
+          "additionalProperties": {},
+          "description": "Custom metadata"
+        }
+      },
+      "required": [
+        "name",
+        "source",
+        "destination"
+      ],
+      "additionalProperties": false
+    }
+  },
+  "$schema": "http://json-schema.org/draft-07/schema#"
+}