fluent-plugin-openlineage 0.1.2 → 0.1.3

data/spec/events/event_invalid_output_dataset_facet.json

@@ -0,0 +1,29 @@
+{
+  "eventType": "COMPLETE",
+  "eventTime": "2020-12-28T19:52:00.001+10:00",
+  "run": {
+    "runId": "41fb5137-f0fd-4ee5-ba5c-56f8571d1bd7"
+  },
+  "job": {
+    "namespace": "my-scheduler-namespace",
+    "name": "myjob"
+  },
+  "inputs": [],
+  "outputs": [
+    {
+      "namespace": "my-datasource-namespace",
+      "name": "instance.schema.output-1",
+      "outputFacets": {
+        "outputStatistics": {
+          "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+          "_schemaURL": "https://openlineage.io/spec/1-0-1/OpenLineage.json#/definitions/OutputStatisticsOutputDatasetFacet",
+          "rowCount": "wrong",
+          "size": 2097152,
+          "fileCount": 5
+        }
+      }
+    }
+  ],
+  "producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+  "schemaURL": "https://openlineage.io/spec/1-0-1/OpenLineage.json#/definitions/RunEvent"
+}

data/spec/events/event_invalid_run_facet.json

@@ -0,0 +1,28 @@
+{
+  "eventType": "COMPLETE",
+  "eventTime": "2020-12-28T19:52:00.001+10:00",
+  "run": {
+    "runId": "41fb5137-f0fd-4ee5-ba5c-56f8571d1bd7",
+    "facets": {
+      "parent": {
+        "run": {
+          "noRunId": "invalid run id"
+        },
+        "job": {
+          "namespace": "parent_namespace",
+          "name": "parent_name"
+        },
+        "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+        "_schemaURL": "https://openlineage.io/spec/1-0-1/OpenLineage.json#/definitions/RunFacet"
+      }
+    }
+  },
+  "job": {
+    "namespace": "my-scheduler-namespace",
+    "name": "myjob"
+  },
+  "inputs": [ ],
+  "outputs": [ ],
+  "producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+  "schemaURL": "https://openlineage.io/spec/1-0-1/OpenLineage.json#/definitions/RunEvent"
+}

data/spec/events/event_no_run_id.json

@@ -0,0 +1,28 @@
+{
+  "eventType": "COMPLETE",
+  "eventTime": "2020-12-28T19:52:00.001+10:00",
+  "run": {
+  },
+  "job": {
+    "namespace": "my-scheduler-namespace",
+    "name": "myjob"
+  },
+  "inputs": [
+    {
+      "namespace": "my-datasource-namespace",
+      "name": "instance.schema.input-1"
+    },
+    {
+      "namespace": "my-datasource-namespace",
+      "name": "instance.schema.input-2"
+    }
+  ],
+  "outputs": [
+    {
+      "namespace": "my-datasource-namespace",
+      "name": "instance.schema.output-1"
+    }
+  ],
+  "producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+  "schemaURL": "https://openlineage.io/spec/1-0-1/OpenLineage.json#/definitions/RunEvent"
+}

data/spec/events/event_simple.json

@@ -0,0 +1,29 @@
+{
+  "eventType": "COMPLETE",
+  "eventTime": "2020-12-28T19:52:00.001+10:00",
+  "run": {
+    "runId": "41fb5137-f0fd-4ee5-ba5c-56f8571d1bd7"
+  },
+  "job": {
+    "namespace": "my-scheduler-namespace",
+    "name": "myjob"
+  },
+  "inputs": [
+    {
+      "namespace": "my-datasource-namespace",
+      "name": "instance.schema.input-1"
+    },
+    {
+      "namespace": "my-datasource-namespace",
+      "name": "instance.schema.input-2"
+    }
+  ],
+  "outputs": [
+    {
+      "namespace": "my-datasource-namespace",
+      "name": "instance.schema.output-1"
+    }
+  ],
+  "producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+  "schemaURL": "https://openlineage.io/spec/1-0-1/OpenLineage.json#/definitions/RunEvent"
+}

data/spec/facets/ColumnLineageDatasetFacet.json

@@ -0,0 +1,96 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-1-0/ColumnLineageDatasetFacet.json",
+  "$defs": {
+    "ColumnLineageDatasetFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/DatasetFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "fields": {
+              "description": "Column level lineage that maps output fields into input fields used to evaluate them.",
+              "type": "object",
+              "additionalProperties": {
+                "type": "object",
+                "properties": {
+                  "inputFields": {
+                    "type": "array",
+                    "items": {
+                      "type": "object",
+                      "properties": {
+                        "namespace": {
+                          "type": "string",
+                          "description": "The input dataset namespace"
+                        },
+                        "name": {
+                          "type": "string",
+                          "description": "The input dataset name"
+                        },
+                        "field": {
+                          "type": "string",
+                          "description": "The input field"
+                        },
+                        "transformations": {
+                          "type": "array",
+                          "items": {
+                            "type": "object",
+                            "properties": {
+                              "type": {
+                                "description": "The type of the transformation. Allowed values are: DIRECT, INDIRECT",
+                                "type": "string"
+                              },
+                              "subtype": {
+                                "type": "string",
+                                "description": "The subtype of the transformation"
+                              },
+                              "description": {
+                                "type": "string",
+                                "description": "a string representation of the transformation applied"
+                              },
+                              "masking": {
+                                "type": "boolean",
+                                "description": "is transformation masking the data or not"
+                              }
+                            },
+                            "required": ["type"],
+                            "additionalProperties": true
+                          }
+                        }
+                      },
+                      "additionalProperties": true,
+                      "required": ["namespace", "name", "field"]
+                    }
+                  },
+                  "transformationDescription": {
+                    "type": "string",
+                    "description": "a string representation of the transformation applied",
+                    "deprecated": true
+                  },
+                  "transformationType": {
+                    "type": "string",
+                    "description": "IDENTITY|MASKED reflects a clearly defined behavior. IDENTITY: exact same as input; MASKED: no original data available (like a hash of PII for example)",
+                    "deprecated": true
+                  }
+                },
+                "additionalProperties": true,
+                "required": ["inputFields"]
+              }
+            }
+          },
+          "additionalProperties": true,
+          "required": ["fields"]
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "columnLineage": {
+      "$ref": "#/$defs/ColumnLineageDatasetFacet"
+    }
+  }
+}

data/spec/facets/ColumnLineageDatasetFacet.md

@@ -0,0 +1,106 @@
+# Column level lineage
+
+The [column level lineage facet](ColumnLineageDatasetFacet.json) captures the lineage of columns of an output dataset
+from the columns in input datasets. It must refer to existing columns as defined in the
+[`schema` facet](SchemaDatasetFacet.json). Additional information on the transformation from the input columns to the
+output column is stored in the optional _transformationDescription_ and _transformationType_ fields.
+
+- _transformationDescription_: a human readable description of the transformation. ex: "(a+b)" or "identical"
+- _transformationType_: type of the transformation. possible values:
+- "IDENTITY" exactly the same as the input.
+- "MASKED" for example a hash of the input value that doesn't expose the original value.
+
+Output Dataset example of adding a columnLineage facet:
+
+```diff
+    {
+      "namespace": "{namespace of the outputdataset}",
+      "name": "{name of the output dataset}",
+      "facets": {
+        "schema": {
+          "fields": [
+            { "name": "{first column of the output dataset}", "type": "{its type}"},
+            { "name": "{second column of the output dataset}", "type": "{its type}"},
+            ...
+          ]
+        },
+>       "columnLineage": {
+>         "fields": {
+>           "{first column of the output dataset}": {
+>             "inputFields": [
+>               { "namespace": "{input dataset namespace}", name: "{input dataset name}", "field": "{input dataset column name}"},
+>               ... other inputs
+>             ],
+>             "transformationDescription": "identical",
+>             "transformationType": "IDENTITY"
+>           },
+>           "{second column of the output dataset}": ...,
+>           ...
+>         }
+>       }
+      }
+    }
+```
+
+Full lineage event example:
+
+```
+{
+  "eventType": "START",
+  "eventTime": "2020-12-09T23:37:31.081Z",
+  "run": {
+    "runId": "3b452093-782c-4ef2-9c0c-aafe2aa6f34d",
+  },
+  "job": {
+    "namespace": "my-scheduler-namespace",
+    "name": "myjob.mytask",
+  },
+  "inputs": [
+    {
+      "namespace": "my-datasource-namespace",
+      "name": "instance.schema.table",
+      "facets": {
+        "schema": {
+          "fields": [
+            { "name": "ia", "type": "INT"},
+            { "name": "ib", "type": "INT"}
+          ]
+        },
+      }
+    }
+  ],
+  "outputs": [
+    {
+      "namespace": "my-datasource-namespace",
+      "name": "instance.schema.output_table",
+      "facets": {
+        "schema": {
+          "fields": [
+            { "name": "a", "type": "INT"},
+            { "name": "b", "type": "INT"}
+          ]
+        },
+        "columnLineage": {
+          "fields": {
+            "a": {
+              "inputFields": [
+                {namespace: "my-datasource-namespace", name: "instance.schema.table", "field": "ia"},
+                ... other inputs
+              ],
+              transformationDescription: "identical",
+              transformationType: "IDENTITY"
+            },
+            "b": ... other output fields
+          }
+        }
+      }
+    }
+  ],
+  "schemaURL": "https://openlineage.io/spec/1-0-0/OpenLineage.json#/definitions/RunEvent"
+}
+```
+
+---
+
+SPDX-License-Identifier: Apache-2.0\
+Copyright 2018-2024 contributors to the OpenLineage project

data/spec/facets/DataQualityAssertionsDatasetFacet.json

@@ -0,0 +1,49 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-1/DataQualityAssertionsDatasetFacet.json",
+  "$defs": {
+    "DataQualityAssertionsDatasetFacet": {
+      "description": "list of tests performed on dataset or dataset columns, and their results",
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/InputDatasetFacet"
+        },
+        {
+          "type": "object",
+          "required": ["assertions"],
+          "properties": {
+            "assertions": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "properties": {
+                  "assertion": {
+                    "type": "string",
+                    "description": "Type of expectation test that dataset is subjected to",
+                    "example": "not_null"
+                  },
+                  "success": {
+                    "type": "boolean"
+                  },
+                  "column": {
+                    "type": "string",
+                    "description": "Column that expectation is testing. It should match the name provided in SchemaDatasetFacet. If column field is empty, then expectation refers to whole dataset.",
+                    "example": "id"
+                  }
+                },
+                "required": ["assertion", "success"]
+              }
+            }
+          }
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "dataQualityAssertions": {
+      "$ref": "#/$defs/DataQualityAssertionsDatasetFacet"
+    }
+  }
+}

data/spec/facets/DataQualityMetricsInputDatasetFacet.json

@@ -0,0 +1,76 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-2/DataQualityMetricsInputDatasetFacet.json",
+  "$defs": {
+    "DataQualityMetricsInputDatasetFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/InputDatasetFacet"
+        },
+        {
+          "type": "object",
+          "required": ["columnMetrics"],
+          "properties": {
+            "rowCount": {
+              "description": "The number of rows evaluated",
+              "type": "integer"
+            },
+            "bytes": {
+              "description": "The size in bytes",
+              "type": "integer"
+            },
+            "fileCount": {
+              "description": "The number of files evaluated",
+              "type": "integer"
+            },
+            "columnMetrics": {
+              "description": "The property key is the column name",
+              "type": "object",
+              "additionalProperties": {
+                "type": "object",
+                "properties": {
+                  "nullCount": {
+                    "description": "The number of null values in this column for the rows evaluated",
+                    "type": "integer"
+                  },
+                  "distinctCount": {
+                    "description": "The number of distinct values in this column for the rows evaluated",
+                    "type": "integer"
+                  },
+                  "sum": {
+                    "description": "The total sum of values in this column for the rows evaluated",
+                    "type": "number"
+                  },
+                  "count": {
+                    "description": "The number of values in this column",
+                    "type": "number"
+                  },
+                  "min": {
+                    "type": "number"
+                  },
+                  "max": {
+                    "type": "number"
+                  },
+                  "quantiles": {
+                    "description": "The property key is the quantile. Examples: 0.1 0.25 0.5 0.75 1",
+                    "type": "object",
+                    "additionalProperties": {
+                      "type": "number"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "dataQualityMetrics": {
+      "$ref": "#/$defs/DataQualityMetricsInputDatasetFacet"
+    }
+  }
+}

data/spec/facets/DatasetVersionDatasetFacet.json

@@ -0,0 +1,31 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-1/DatasetVersionDatasetFacet.json",
+  "$defs": {
+    "DatasetVersionDatasetFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/DatasetFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "datasetVersion": {
+              "description": "The version of the dataset.",
+              "type": "string"
+            }
+          },
+          "additionalProperties": true,
+          "required": ["datasetVersion"]
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "version": {
+      "$ref": "#/$defs/DatasetVersionDatasetFacet"
+    }
+  }
+}

data/spec/facets/DatasourceDatasetFacet.json

@@ -0,0 +1,32 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-1/DatasourceDatasetFacet.json",
+  "$defs": {
+    "DatasourceDatasetFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/DatasetFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "name": {
+              "type": "string"
+            },
+            "uri": {
+              "type": "string",
+              "format": "uri"
+            }
+          }
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "dataSource": {
+      "$ref": "#/$defs/DatasourceDatasetFacet"
+    }
+  }
+}

data/spec/facets/DocumentationDatasetFacet.json

@@ -0,0 +1,31 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-1/DocumentationDatasetFacet.json",
+  "$defs": {
+    "DocumentationDatasetFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/DatasetFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "description": {
+              "description": "The description of the dataset.",
+              "type": "string",
+              "example": "canonical representation of entity Foo"
+            }
+          },
+          "required": ["description"]
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "documentation": {
+      "$ref": "#/$defs/DocumentationDatasetFacet"
+    }
+  }
+}

data/spec/facets/DocumentationJobFacet.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-1/DocumentationJobFacet.json",
+  "$defs": {
+    "DocumentationJobFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/JobFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "description": {
+              "description": "The description of the job.",
+              "type": "string"
+            }
+          },
+          "required": ["description"]
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "documentation": {
+      "$ref": "#/$defs/DocumentationJobFacet"
+    }
+  }
+}

data/spec/facets/ErrorMessageRunFacet.json

@@ -0,0 +1,41 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-1/ErrorMessageRunFacet.json",
+  "$defs": {
+    "ErrorMessageRunFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "message": {
+              "description": "A human-readable string representing error message generated by observed system",
+              "type": "string",
+              "example": "org.apache.spark.sql.AnalysisException: Table or view not found: wrong_table_name; line 1 pos 14"
+            },
+            "programmingLanguage": {
+              "description": "Programming language the observed system uses.",
+              "type": "string",
+              "example": "JAVA"
+            },
+            "stackTrace": {
+              "description": "A language-specific stack trace generated by observed system",
+              "type": "string",
+              "example": "Exception in thread \"main\" java.lang.RuntimeException: A test exception\nat io.openlineage.SomeClass.method(SomeClass.java:13)\nat io.openlineage.SomeClass.anotherMethod(SomeClass.java:9)"
+            }
+          },
+          "required": ["message", "programmingLanguage"]
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "errorMessage": {
+      "$ref": "#/$defs/ErrorMessageRunFacet"
+    }
+  }
+}

data/spec/facets/ExternalQueryRunFacet.json

@@ -0,0 +1,36 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-2/ExternalQueryRunFacet.json",
+  "$defs": {
+    "ExternalQueryRunFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "externalQueryId": {
+              "description": "Identifier for the external system",
+              "type": "string",
+              "example": "my-project-1234:US.bquijob_123x456_123y123z123c"
+            },
+            "source": {
+              "description": "source of the external query",
+              "type": "string",
+              "example": "bigquery"
+            }
+          },
+          "required": ["externalQueryId", "source"]
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "externalQuery": {
+      "$ref": "#/$defs/ExternalQueryRunFacet"
+    }
+  }
+}