fluent-plugin-openlineage 0.1.2 → 0.1.3

data/spec/facets/ExternalQueryRunFacet.md

@@ -0,0 +1,49 @@
+# External Query Run
+
+The [external query run facet](ExternalQueryRunFacet.json) captures the identifier of the query that ran on an external
+source systems such as [bigquery](https://cloud.google.com/bigquery).
+
+_externalQueryId_ and _source_ fields.
+
+- _externalQueryId_: unique identifier of the query from the source system.
+- _source_: type of the source (e.g. bigquery)
+
+Full lineage event example:
+
+```
+{
+  "eventType": "START",
+  "eventTime": "2020-12-09T23:37:31.081Z",
+  "run": {
+    "runId": "3b452093-782c-4ef2-9c0c-aafe2aa6f34d",
+    "facets": {
+>        "externalQuery": {
+>            "externalQueryId": "my-project-1234:US.bquijob_123x456_123y123z123c",
+>            "source": "bigquery"
+>        }
+    }
+  },
+  "job": {
+    "namespace": "my-namespace",
+    "name": "myjob.mytask",
+  },
+  "inputs": [
+    {
+      "namespace": "my-datasource-namespace",
+      "name": "instance.schema.table",
+    }
+  ],
+  "outputs": [
+    {
+      "namespace": "my-datasource-namespace",
+      "name": "instance.schema.output_table",
+    }
+  ],
+  "schemaURL": "https://openlineage.io/spec/1-0-0/OpenLineage.json#/definitions/RunEvent"
+}
+```
+
+---
+
+SPDX-License-Identifier: Apache-2.0\
+Copyright 2018-2024 contributors to the OpenLineage project

data/spec/facets/ExtractionErrorRunFacet.json

@@ -0,0 +1,58 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-1-2/ExtractionErrorRunFacet.json",
+  "$defs": {
+    "ExtractionErrorRunFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "totalTasks": {
+              "description": "The number of distinguishable tasks in a run that were processed by OpenLineage, whether successfully or not. Those could be, for example, distinct SQL statements.",
+              "type": "integer"
+            },
+            "failedTasks": {
+              "description": "The number of distinguishable tasks in a run that were processed not successfully by OpenLineage. Those could be, for example, distinct SQL statements.",
+              "type": "integer"
+            },
+            "errors": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "properties": {
+                  "errorMessage": {
+                    "description": "Text representation of extraction error message.",
+                    "type": "string"
+                  },
+                  "stackTrace": {
+                    "description": "Stack trace of extraction error message",
+                    "type": "string"
+                  },
+                  "task": {
+                    "description": "Text representation of task that failed. This can be, for example, SQL statement that parser could not interpret.",
+                    "type": "string"
+                  },
+                  "taskNumber": {
+                    "description": "Order of task (counted from 0).",
+                    "type": "integer"
+                  }
+                },
+                "required": ["errorMessage"]
+              }
+            }
+          },
+          "required": ["totalTasks", "failedTasks", "errors"]
+        }
+      ]
+    }
+  },
+  "type": "object",
+  "properties": {
+    "extractionError": {
+      "$ref": "#/$defs/ExtractionErrorRunFacet"
+    }
+  }
+}

data/spec/facets/JobTypeJobFacet.json

@@ -0,0 +1,41 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/2-0-3/JobTypeJobFacet.json",
+  "$defs": {
+    "JobTypeJobFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/JobFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "processingType": {
+              "description": "Job processing type like: BATCH or STREAMING",
+              "type": "string",
+              "example": "BATCH"
+            },
+            "integration": {
+              "description": "OpenLineage integration type of this job: for example SPARK|DBT|AIRFLOW|FLINK",
+              "type": "string",
+              "example": "SPARK"
+            },
+            "jobType": {
+              "description": "Run type, for example: QUERY|COMMAND|DAG|TASK|JOB|MODEL. This is an integration-specific field.",
+              "type": "string",
+              "example": "QUERY"
+            }
+          },
+          "required": ["processingType", "integration"]
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "jobType": {
+      "$ref": "#/$defs/JobTypeJobFacet"
+    }
+  }
+}

data/spec/facets/LifecycleStateChangeDatasetFacet.json

@@ -0,0 +1,46 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-1/LifecycleStateChangeDatasetFacet.json",
+  "$defs": {
+    "LifecycleStateChangeDatasetFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/DatasetFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "lifecycleStateChange": {
+              "description": "The lifecycle state change.",
+              "type": "string",
+              "enum": ["ALTER", "CREATE", "DROP", "OVERWRITE", "RENAME", "TRUNCATE"]
+            },
+            "previousIdentifier": {
+              "description": "Previous name of the dataset in case of renaming it.",
+              "type": "object",
+              "properties": {
+                "name": {
+                  "documentation": "Previous dataset name.",
+                  "type": "string"
+                },
+                "namespace": {
+                  "documentation": "Previous dataset namespace.",
+                  "type": "string"
+                }
+              },
+              "required": ["name", "namespace"]
+            }
+          },
+          "required": ["lifecycleStateChange"]
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "lifecycleStateChange": {
+      "$ref": "#/$defs/LifecycleStateChangeDatasetFacet"
+    }
+  }
+}

data/spec/facets/NominalTimeRunFacet.json

@@ -0,0 +1,38 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-1/NominalTimeRunFacet.json",
+  "$defs": {
+    "NominalTimeRunFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "nominalStartTime": {
+              "description": "An [ISO-8601](https://en.wikipedia.org/wiki/ISO_8601) timestamp representing the nominal start time (included) of the run. AKA the schedule time",
+              "type": "string",
+              "format": "date-time",
+              "example": "2020-12-17T03:00:00.000Z"
+            },
+            "nominalEndTime": {
+              "description": "An [ISO-8601](https://en.wikipedia.org/wiki/ISO_8601) timestamp representing the nominal end time (excluded) of the run. (Should be the nominal start time of the next run)",
+              "type": "string",
+              "format": "date-time",
+              "example": "2020-12-17T04:00:00.000Z"
+            }
+          },
+          "required": ["nominalStartTime"]
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "nominalTime": {
+      "$ref": "#/$defs/NominalTimeRunFacet"
+    }
+  }
+}

data/spec/facets/OutputStatisticsOutputDatasetFacet.json

@@ -0,0 +1,36 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-2/OutputStatisticsOutputDatasetFacet.json",
+  "$defs": {
+    "OutputStatisticsOutputDatasetFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/OutputDatasetFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "rowCount": {
+              "description": "The number of rows written to the dataset",
+              "type": "integer"
+            },
+            "size": {
+              "description": "The size in bytes written to the dataset",
+              "type": "integer"
+            },
+            "fileCount": {
+              "description": "The number of files written to the dataset",
+              "type": "integer"
+            }
+          }
+        }
+      ]
+    }
+  },
+  "type": "object",
+  "properties": {
+    "outputStatistics": {
+      "$ref": "#/$defs/OutputStatisticsOutputDatasetFacet"
+    }
+  }
+}

data/spec/facets/OwnershipDatasetFacet.json

@@ -0,0 +1,45 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-1/OwnershipDatasetFacet.json",
+  "$defs": {
+    "OwnershipDatasetFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/DatasetFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "owners": {
+              "description": "The owners of the dataset.",
+              "type": "array",
+              "items": {
+                "type": "object",
+                "properties": {
+                  "name": {
+                    "description": "the identifier of the owner of the Dataset. It is recommended to define this as a URN. For example application:foo, user:jdoe, team:data",
+                    "type": "string",
+                    "example": "application:app_name"
+                  },
+                  "type": {
+                    "description": "The type of ownership (optional)",
+                    "type": "string",
+                    "example": "MAINTAINER"
+                  }
+                },
+                "required": ["name"]
+              }
+            }
+          }
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "ownership": {
+      "$ref": "#/$defs/OwnershipDatasetFacet"
+    }
+  }
+}

data/spec/facets/OwnershipJobFacet.json

@@ -0,0 +1,45 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-1/OwnershipJobFacet.json",
+  "$defs": {
+    "OwnershipJobFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/JobFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "owners": {
+              "description": "The owners of the job.",
+              "type": "array",
+              "items": {
+                "type": "object",
+                "properties": {
+                  "name": {
+                    "description": "the identifier of the owner of the Job. It is recommended to define this as a URN. For example application:foo, user:jdoe, team:data",
+                    "type": "string",
+                    "example": "application:app_name"
+                  },
+                  "type": {
+                    "description": "The type of ownership (optional)",
+                    "type": "string",
+                    "example": "MAINTAINER"
+                  }
+                },
+                "required": ["name"]
+              }
+            }
+          }
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "ownership": {
+      "$ref": "#/$defs/OwnershipJobFacet"
+    }
+  }
+}

data/spec/facets/ParentRunFacet.json

@@ -0,0 +1,54 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-1/ParentRunFacet.json",
+  "$defs": {
+    "ParentRunFacet": {
+      "description": "the id of the parent run and job, iff this run was spawn from an other run (for example, the Dag run scheduling its tasks)",
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "run": {
+              "type": "object",
+              "properties": {
+                "runId": {
+                  "description": "The globally unique ID of the run associated with the job.",
+                  "type": "string",
+                  "format": "uuid"
+                }
+              },
+              "required": ["runId"]
+            },
+            "job": {
+              "type": "object",
+              "properties": {
+                "namespace": {
+                  "description": "The namespace containing that job",
+                  "type": "string",
+                  "example": "my-scheduler-namespace"
+                },
+                "name": {
+                  "description": "The unique name for that job within that namespace",
+                  "type": "string",
+                  "example": "myjob.mytask"
+                }
+              },
+              "required": ["namespace", "name"]
+            }
+          },
+          "required": ["run", "job"]
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "parent": {
+      "$ref": "#/$defs/ParentRunFacet"
+    }
+  }
+}

data/spec/facets/ProcessingEngineRunFacet.json

@@ -0,0 +1,41 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-1-1/ProcessingEngineRunFacet.json",
+  "$defs": {
+    "ProcessingEngineRunFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "version": {
+              "description": "Processing engine version. Might be Airflow or Spark version.",
+              "type": "string",
+              "example": "2.5.0"
+            },
+            "name": {
+              "description": "Processing engine name, e.g. Airflow or Spark",
+              "type": "string",
+              "example": "Airflow"
+            },
+            "openlineageAdapterVersion": {
+              "description": "OpenLineage adapter package version. Might be e.g. OpenLineage Airflow integration package version",
+              "type": "string",
+              "example": "0.19.0"
+            }
+          },
+          "required": ["version"]
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "processing_engine": {
+      "$ref": "#/$defs/ProcessingEngineRunFacet"
+    }
+  }
+}

data/spec/facets/SQLJobFacet.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-1/SQLJobFacet.json",
+  "$defs": {
+    "SQLJobFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/JobFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "query": {
+              "type": "string",
+              "example": "SELECT * FROM foo"
+            }
+          },
+          "required": ["query"]
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "sql": {
+      "$ref": "#/$defs/SQLJobFacet"
+    }
+  }
+}

data/spec/facets/SchemaDatasetFacet.json

@@ -0,0 +1,59 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-1-1/SchemaDatasetFacet.json",
+  "$defs": {
+    "SchemaDatasetFacetFields": {
+      "type": "object",
+      "properties": {
+        "name": {
+          "description": "The name of the field.",
+          "type": "string",
+          "example": "column1"
+        },
+        "type": {
+          "description": "The type of the field.",
+          "type": "string",
+          "example": "VARCHAR|INT|..."
+        },
+        "description": {
+          "description": "The description of the field.",
+          "type": "string"
+        },
+        "fields": {
+          "description": "Nested struct fields.",
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/SchemaDatasetFacetFields"
+          }
+        }
+      },
+      "required": ["name"]
+    },
+    "SchemaDatasetFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/DatasetFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "fields": {
+              "description": "The fields of the data source.",
+              "type": "array",
+              "items": {
+                "$ref": "#/$defs/SchemaDatasetFacetFields"
+              }
+            }
+          }
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "schema": {
+      "$ref": "#/$defs/SchemaDatasetFacet"
+    }
+  }
+}

data/spec/facets/SourceCodeJobFacet.json

@@ -0,0 +1,34 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-1/SourceCodeJobFacet.json",
+  "$defs": {
+    "SourceCodeJobFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/JobFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "language": {
+              "description": "Language in which source code of this job was written.",
+              "type": "string"
+            },
+            "sourceCode": {
+              "description": "Source code of this job.",
+              "type": "string"
+            }
+          },
+          "required": ["language", "sourceCode"]
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "sourceCode": {
+      "$ref": "#/$defs/SourceCodeJobFacet"
+    }
+  }
+}

data/spec/facets/SourceCodeLocationJobFacet.json

@@ -0,0 +1,60 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-1/SourceCodeLocationJobFacet.json",
+  "$defs": {
+    "SourceCodeLocationJobFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/JobFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "type": {
+              "type": "string",
+              "description": "the source control system",
+              "example": "git|svn"
+            },
+            "url": {
+              "type": "string",
+              "description": "the full http URL to locate the file",
+              "format": "uri",
+              "example": "https://github.com/MarquezProject/marquez-airflow-quickstart/blob/693e35482bc2e526ced2b5f9f76ef83dec6ec691/dags/dummy_example.py"
+            },
+            "repoUrl": {
+              "type": "string",
+              "description": "the URL to the repository",
+              "example": "git@github.com:{org}/{repo}.git or https://github.com/{org}/{repo}.git|svn://<your_ip>/<repository_name>"
+            },
+            "path": {
+              "type": "string",
+              "description": "the path in the repo containing the source files",
+              "example": "path/to/my/dags"
+            },
+            "version": {
+              "type": "string",
+              "description": "the current version deployed (not a branch name, the actual unique version)",
+              "example": "git: the git sha | Svn: the revision number"
+            },
+            "tag": {
+              "type": "string",
+              "description": "optional tag name"
+            },
+            "branch": {
+              "type": "string",
+              "description": "optional branch name"
+            }
+          },
+          "required": ["type", "url"]
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "sourceCodeLocation": {
+      "$ref": "#/$defs/SourceCodeLocationJobFacet"
+    }
+  }
+}