@loxtep/sdk 0.7.22 → 0.7.24

package/schemas/entity-json-schemas/transformation.json

@@ -0,0 +1,166 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://loxtep.io/schemas/entity/transformation.json",
+  "title": "Transformation Entity",
+  "description": "JSON Schema for Transformation entity in customer workspace. Transformations are pure functions that mutate/enrich events within a workflow flow. Each transformation references its workflow via workflow_id and its upstream entity (connection or previous transformation) via upstream_entity_id.",
+  "type": "object",
+  "required": [
+    "transformation_id",
+    "organization_id",
+    "project_id",
+    "workflow_id",
+    "name",
+    "transform_type",
+    "operation_config",
+    "created_at",
+    "updated_at"
+  ],
+  "properties": {
+    "transformation_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Unique identifier for the transformation"
+    },
+    "organization_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Organization that owns this transformation"
+    },
+    "project_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Project that contains this transformation"
+    },
+    "workflow_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Workflow this transformation belongs to"
+    },
+    "upstream_entity_id": {
+      "type": ["string", "null"],
+      "format": "uuid",
+      "description": "Reference to the upstream entity in the flow graph (connection or previous transformation). Null only when this is the first transformation directly after the trigger connection."
+    },
+    "upstream_entity_type": {
+      "type": ["string", "null"],
+      "enum": ["connections", "transformations", null],
+      "description": "Entity type of the upstream entity (connections or transformations)"
+    },
+    "name": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 255,
+      "description": "Transformation name"
+    },
+    "description": {
+      "type": ["string", "null"],
+      "maxLength": 1000,
+      "description": "Transformation description"
+    },
+    "transform_type": {
+      "type": "string",
+      "enum": [
+        "drop_fields",
+        "select_fields",
+        "rename_field",
+        "drop_null_fields",
+        "drop_duplicates",
+        "flatten",
+        "explode_array",
+        "array_to_column",
+        "capitalize",
+        "format_timestamp",
+        "add_current_timestamp",
+        "parse_json",
+        "xml_to_json",
+        "extract_json_path",
+        "regex_extractor",
+        "concatenate_columns",
+        "derived_columns",
+        "fill_missing_values",
+        "detect_sensitive_data",
+        "evaluate_data_quality",
+        "change_schema",
+        "filter",
+        "batch",
+        "delay",
+        "split",
+        "execute_code_function",
+        "crypto",
+        "formatter",
+        "assign_identifier",
+        "remove_null_events",
+        "map",
+        "enrich",
+        "add_fields",
+        "rename_fields",
+        "custom"
+      ],
+      "description": "Type of transformation operation"
+    },
+    "operation_config": {
+      "type": "object",
+      "additionalProperties": true,
+      "description": "Transformation-specific configuration parameters",
+      "examples": [
+        {
+          "fields": ["field1", "field2"]
+        },
+        {
+          "condition": "status === 'active'"
+        },
+        {
+          "field": "name",
+          "case": "upper"
+        }
+      ]
+    },
+    "input_schema": {
+      "type": ["object", "null"],
+      "description": "Optional: JSON Schema of input events (for validation and type checking). Any JSON Schema object.",
+      "additionalProperties": true
+    },
+    "output_schema": {
+      "type": ["object", "null"],
+      "description": "Optional: JSON Schema of output events after transformation (typically inferred). Any JSON Schema object.",
+      "additionalProperties": true
+    },
+    "order": {
+      "type": ["integer", "null"],
+      "minimum": 0,
+      "description": "Default execution order when used in workflows (lower = earlier)"
+    },
+    "metadata": {
+      "type": "object",
+      "additionalProperties": true,
+      "default": {},
+      "description": "Additional metadata for the transformation"
+    },
+    "created_by": {
+      "type": ["string", "null"],
+      "format": "uuid",
+      "description": "User ID who created the transformation"
+    },
+    "updated_by": {
+      "type": ["string", "null"],
+      "format": "uuid",
+      "description": "User ID who last updated the transformation"
+    },
+    "created_at": {
+      "type": "string",
+      "format": "date-time",
+      "description": "ISO 8601 timestamp when transformation was created"
+    },
+    "updated_at": {
+      "type": "string",
+      "format": "date-time",
+      "description": "ISO 8601 timestamp when transformation was last updated"
+    },
+    "deleted_at": {
+      "type": ["string", "null"],
+      "format": "date-time",
+      "description": "ISO 8601 timestamp when transformation was deleted (soft delete)"
+    }
+  },
+  "additionalProperties": false
+}

package/schemas/entity-json-schemas/validation.json

@@ -0,0 +1,152 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://loxtep.io/schemas/entity/validation.json",
+  "title": "Validation Entity",
+  "description": "JSON Schema for Validation entity in customer workspace. Validations are rules that assert data quality and reject invalid events within a workflow flow. Each validation references its workflow via workflow_id and the upstream entity it applies to via upstream_entity_id.",
+  "type": "object",
+  "required": [
+    "validation_id",
+    "organization_id",
+    "project_id",
+    "workflow_id",
+    "name",
+    "validation_type",
+    "rules",
+    "severity",
+    "created_at",
+    "updated_at"
+  ],
+  "properties": {
+    "validation_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Unique identifier for the validation"
+    },
+    "organization_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Organization that owns this validation"
+    },
+    "project_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Project that contains this validation"
+    },
+    "workflow_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Workflow this validation belongs to"
+    },
+    "upstream_entity_id": {
+      "type": ["string", "null"],
+      "format": "uuid",
+      "description": "Reference to the upstream entity this validation applies to (connection, transformation, or data product)"
+    },
+    "upstream_entity_type": {
+      "type": ["string", "null"],
+      "enum": ["connections", "transformations", "data-products", null],
+      "description": "Entity type of the upstream entity"
+    },
+    "name": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 255,
+      "description": "Validation name"
+    },
+    "description": {
+      "type": ["string", "null"],
+      "maxLength": 1000,
+      "description": "Validation description"
+    },
+    "validation_type": {
+      "type": "string",
+      "enum": [
+        "schema",
+        "business_rule",
+        "data_quality",
+        "pii_check",
+        "completeness",
+        "format"
+      ],
+      "description": "Type of validation"
+    },
+    "schema_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Org domain schema reference; deploy resolves latest active version into rules.schema when rules.schema is empty"
+    },
+    "schema_version_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Pinned org domain schema version; deploy resolves definition into rules.schema when rules.schema is empty"
+    },
+    "rules": {
+      "type": "object",
+      "additionalProperties": true,
+      "description": "Validation rules (structure depends on validation_type). For validation_type schema, rules.schema holds the JSON Schema; rules.schema_id may reference org registry instead.",
+      "examples": [
+        {
+          "schema": {
+            "type": "object",
+            "description": "JSON Schema for validation"
+          }
+        },
+        {
+          "condition": "field > 100",
+          "message": "Field must be greater than 100"
+        },
+        {
+          "required_fields": ["email", "name"],
+          "completeness_threshold": 0.95
+        }
+      ]
+    },
+    "severity": {
+      "type": "string",
+      "enum": ["error", "warning", "info"],
+      "description": "Severity level when validation fails"
+    },
+    "error_message": {
+      "type": ["string", "null"],
+      "maxLength": 500,
+      "description": "Custom error message when validation fails"
+    },
+    "order": {
+      "type": ["integer", "null"],
+      "minimum": 0,
+      "description": "Default execution order when used in workflows (lower = earlier)"
+    },
+    "metadata": {
+      "type": "object",
+      "additionalProperties": true,
+      "default": {},
+      "description": "Additional metadata for the validation"
+    },
+    "created_by": {
+      "type": ["string", "null"],
+      "format": "uuid",
+      "description": "User ID who created the validation"
+    },
+    "updated_by": {
+      "type": ["string", "null"],
+      "format": "uuid",
+      "description": "User ID who last updated the validation"
+    },
+    "created_at": {
+      "type": "string",
+      "format": "date-time",
+      "description": "ISO 8601 timestamp when validation was created"
+    },
+    "updated_at": {
+      "type": "string",
+      "format": "date-time",
+      "description": "ISO 8601 timestamp when validation was last updated"
+    },
+    "deleted_at": {
+      "type": ["string", "null"],
+      "format": "date-time",
+      "description": "ISO 8601 timestamp when validation was deleted (soft delete)"
+    }
+  },
+  "additionalProperties": false
+}

package/schemas/entity-json-schemas/workflow-graph.json

@@ -0,0 +1,116 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://loxtep.io/schemas/workflow-graph.json",
+  "title": "Workflow Graph",
+  "description": "Materialized read-only graph view for a workflow (flow). Built from per-entity S3 files; not a writable entity type.",
+  "type": "object",
+  "required": [
+    "workflow_id",
+    "organization_id",
+    "project_id",
+    "version",
+    "built_at",
+    "workflow",
+    "nodes",
+    "edges",
+    "connectors",
+    "deployment",
+    "metadata"
+  ],
+  "properties": {
+    "workflow_id": { "type": "string", "format": "uuid" },
+    "organization_id": { "type": "string", "format": "uuid" },
+    "project_id": { "type": "string", "format": "uuid" },
+    "version": { "type": "integer", "minimum": 1 },
+    "built_at": { "type": "string", "format": "date-time" },
+    "applied_template_digest": { "type": ["string", "null"] },
+    "workflow": {
+      "type": "object",
+      "description": "Full workflow.json entity snapshot"
+    },
+    "nodes": {
+      "type": "object",
+      "additionalProperties": {
+        "type": "object",
+        "required": ["entity_type", "entity_id", "data"],
+        "properties": {
+          "entity_type": { "type": "string" },
+          "entity_id": { "type": "string", "format": "uuid" },
+          "upstream_entity_id": { "type": ["string", "null"] },
+          "deployment_status": {
+            "type": "string",
+            "enum": ["undeployed", "deployed", "stale"]
+          },
+          "container_id": { "type": ["string", "null"] },
+          "data": { "type": "object" }
+        }
+      }
+    },
+    "edges": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["from", "to", "type"],
+        "properties": {
+          "from": { "type": "string" },
+          "to": { "type": "string" },
+          "type": { "type": "string", "enum": ["upstream"] }
+        }
+      }
+    },
+    "connectors": {
+      "type": "object",
+      "additionalProperties": { "type": "object" }
+    },
+    "deployment": {
+      "type": "object",
+      "required": [
+        "deployment_id",
+        "deployed_at",
+        "version_id",
+        "microservice_id",
+        "runtime_mapping"
+      ],
+      "properties": {
+        "deployment_id": { "type": ["string", "null"], "format": "uuid" },
+        "deployed_at": { "type": ["string", "null"], "format": "date-time" },
+        "version_id": { "type": ["string", "null"], "format": "uuid" },
+        "microservice_id": { "type": ["string", "null"] },
+        "runtime_mapping": {
+          "type": "object",
+          "properties": {
+            "containers": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "properties": {
+                  "container_id": { "type": "string" },
+                  "entity_id": { "type": "string" },
+                  "entity_type": { "type": "string" },
+                  "bot_ids": { "type": "array", "items": { "type": "string" } },
+                  "queue_ids": {
+                    "type": "array",
+                    "items": { "type": "string" }
+                  }
+                }
+              }
+            },
+            "all_bot_ids": { "type": "array", "items": { "type": "string" } },
+            "all_queue_ids": { "type": "array", "items": { "type": "string" } }
+          }
+        }
+      }
+    },
+    "metadata": {
+      "type": "object",
+      "properties": {
+        "node_count": { "type": "integer" },
+        "entity_type_counts": {
+          "type": "object",
+          "additionalProperties": { "type": "integer" }
+        },
+        "requires_redeployment": { "type": "boolean" }
+      }
+    }
+  }
+}

package/schemas/entity-json-schemas/workflow.json

@@ -0,0 +1,193 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://loxtep.io/schemas/entity/workflow.json",
+  "title": "Workflow Entity",
+  "description": "JSON Schema for the root workflow entity in customer workspace. A workflow is a flow. Flow nodes (connection, transformation, validation, data product) reference the workflow via workflow_id and point upstream via upstream_entity_id; the workflow does not store node IDs. template_id identifies the flow starter template used to create this flow (e.g. blank, webhook-ingestion), distinct from connector catalog templates.",
+  "type": "object",
+  "required": [
+    "workflow_id",
+    "organization_id",
+    "project_id",
+    "name",
+    "template_id",
+    "workflow_type",
+    "domain_id",
+    "status",
+    "created_at",
+    "updated_at"
+  ],
+  "properties": {
+    "workflow_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Unique identifier for the workflow"
+    },
+    "organization_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Organization that owns this workflow"
+    },
+    "project_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Project that contains this workflow"
+    },
+    "name": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 255,
+      "description": "Workflow display name"
+    },
+    "description": {
+      "type": ["string", "null"],
+      "maxLength": 1000,
+      "description": "Workflow description"
+    },
+    "template_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Workflow starter template identifier used when this flow was created (e.g. blank, webhook-ingestion). Distinct from connector catalog templates."
+    },
+    "workflow_type": {
+      "type": "string",
+      "enum": ["ingestion", "enrichment", "consumption"],
+      "description": "Type of flow: ingestion (ingest from external sources), enrichment (transform/join data), consumption (expose or consume data products)."
+    },
+    "domain_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Domain this workflow belongs to (required)."
+    },
+    "segments": {
+      "type": "array",
+      "default": [],
+      "description": "Visual groupings of nodes in the graph editor.",
+      "items": {
+        "type": "object",
+        "properties": {
+          "segment_id": {
+            "type": "string",
+            "description": "Unique identifier for the segment"
+          },
+          "name": { "type": "string", "description": "Display name" },
+          "description": {
+            "type": ["string", "null"],
+            "description": "Optional description"
+          },
+          "node_ids": {
+            "type": "array",
+            "items": { "type": "string" },
+            "description": "Entity IDs of nodes in this segment"
+          },
+          "color": {
+            "type": ["string", "null"],
+            "description": "Optional color for UI"
+          }
+        },
+        "additionalProperties": false
+      }
+    },
+    "trigger": {
+      "type": ["object", "null"],
+      "description": "Workflow trigger configuration (event, schedule, or connector).",
+      "properties": {
+        "event": {
+          "type": ["string", "null"],
+          "description": "Event-based trigger (e.g. webhook, queue)"
+        },
+        "schedule": {
+          "type": ["string", "null"],
+          "description": "Cron or interval for scheduled execution"
+        },
+        "connector": {
+          "type": ["string", "null"],
+          "format": "uuid",
+          "description": "Connector ID for connector-driven triggers"
+        }
+      },
+      "additionalProperties": false
+    },
+    "configuration": {
+      "type": "object",
+      "description": "Workflow configuration (schedule, batch_size, etc.). Flow nodes are discovered by workflow_id and upstream_entity_id; the workflow does not store connection_id or transformation_ids.",
+      "properties": {
+        "schedule": {
+          "type": ["string", "null"],
+          "description": "Cron expression for scheduled execution (optional)"
+        },
+        "batch_size": {
+          "type": ["integer", "null"],
+          "minimum": 1,
+          "description": "Number of events to process per batch"
+        }
+      },
+      "additionalProperties": false
+    },
+    "deployment": {
+      "type": ["object", "null"],
+      "description": "Deployment information (set when workflow is deployed)",
+      "properties": {
+        "instance_id": {
+          "type": "string",
+          "format": "uuid"
+        },
+        "lambda_functions": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "queues": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      },
+      "additionalProperties": false
+    },
+    "status": {
+      "type": "string",
+      "enum": ["active", "paused", "error", "pending", "inactive"],
+      "description": "Workflow status"
+    },
+    "metrics": {
+      "type": "object",
+      "additionalProperties": true,
+      "default": {},
+      "description": "Workflow metrics (events processed, error rate, etc.)"
+    },
+    "metadata": {
+      "type": "object",
+      "additionalProperties": true,
+      "default": {},
+      "description": "Additional metadata for the workflow"
+    },
+    "created_by": {
+      "type": ["string", "null"],
+      "format": "uuid",
+      "description": "User ID who created the workflow"
+    },
+    "updated_by": {
+      "type": ["string", "null"],
+      "format": "uuid",
+      "description": "User ID who last updated the workflow"
+    },
+    "created_at": {
+      "type": "string",
+      "format": "date-time",
+      "description": "ISO 8601 timestamp when workflow was created"
+    },
+    "updated_at": {
+      "type": "string",
+      "format": "date-time",
+      "description": "ISO 8601 timestamp when workflow was last updated"
+    },
+    "deleted_at": {
+      "type": ["string", "null"],
+      "format": "date-time",
+      "description": "ISO 8601 timestamp when workflow was deleted (soft delete)"
+    }
+  },
+  "additionalProperties": false
+}