fluentd-openlineage-parser 0.1.0

data/spec/facets/JobTypeJobFacet.json

@@ -0,0 +1,41 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/2-0-3/JobTypeJobFacet.json",
+  "$defs": {
+    "JobTypeJobFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/JobFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "processingType": {
+              "description": "Job processing type like: BATCH or STREAMING",
+              "type": "string",
+              "example": "BATCH"
+            },
+            "integration": {
+              "description": "OpenLineage integration type of this job: for example SPARK|DBT|AIRFLOW|FLINK",
+              "type": "string",
+              "example": "SPARK"
+            },
+            "jobType": {
+              "description": "Run type, for example: QUERY|COMMAND|DAG|TASK|JOB|MODEL. This is an integration-specific field.",
+              "type": "string",
+              "example": "QUERY"
+            }
+          },
+          "required": ["processingType", "integration"]
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "jobType": {
+      "$ref": "#/$defs/JobTypeJobFacet"
+    }
+  }
+}

data/spec/facets/LifecycleStateChangeDatasetFacet.json

@@ -0,0 +1,46 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-1/LifecycleStateChangeDatasetFacet.json",
+  "$defs": {
+    "LifecycleStateChangeDatasetFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/DatasetFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "lifecycleStateChange": {
+              "description": "The lifecycle state change.",
+              "type": "string",
+              "enum": ["ALTER", "CREATE", "DROP", "OVERWRITE", "RENAME", "TRUNCATE"]
+            },
+            "previousIdentifier": {
+              "description": "Previous name of the dataset in case of renaming it.",
+              "type": "object",
+              "properties": {
+                "name": {
+                  "documentation": "Previous dataset name.",
+                  "type": "string"
+                },
+                "namespace": {
+                  "documentation": "Previous dataset namespace.",
+                  "type": "string"
+                }
+              },
+              "required": ["name", "namespace"]
+            }
+          },
+          "required": ["lifecycleStateChange"]
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "lifecycleStateChange": {
+      "$ref": "#/$defs/LifecycleStateChangeDatasetFacet"
+    }
+  }
+}

data/spec/facets/NominalTimeRunFacet.json

@@ -0,0 +1,38 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-1/NominalTimeRunFacet.json",
+  "$defs": {
+    "NominalTimeRunFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "nominalStartTime": {
+              "description": "An [ISO-8601](https://en.wikipedia.org/wiki/ISO_8601) timestamp representing the nominal start time (included) of the run. AKA the schedule time",
+              "type": "string",
+              "format": "date-time",
+              "example": "2020-12-17T03:00:00.000Z"
+            },
+            "nominalEndTime": {
+              "description": "An [ISO-8601](https://en.wikipedia.org/wiki/ISO_8601) timestamp representing the nominal end time (excluded) of the run. (Should be the nominal start time of the next run)",
+              "type": "string",
+              "format": "date-time",
+              "example": "2020-12-17T04:00:00.000Z"
+            }
+          },
+          "required": ["nominalStartTime"]
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "nominalTime": {
+      "$ref": "#/$defs/NominalTimeRunFacet"
+    }
+  }
+}

data/spec/facets/OutputStatisticsOutputDatasetFacet.json

@@ -0,0 +1,36 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-2/OutputStatisticsOutputDatasetFacet.json",
+  "$defs": {
+    "OutputStatisticsOutputDatasetFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/OutputDatasetFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "rowCount": {
+              "description": "The number of rows written to the dataset",
+              "type": "integer"
+            },
+            "size": {
+              "description": "The size in bytes written to the dataset",
+              "type": "integer"
+            },
+            "fileCount": {
+              "description": "The number of files written to the dataset",
+              "type": "integer"
+            }
+          }
+        }
+      ]
+    }
+  },
+  "type": "object",
+  "properties": {
+    "outputStatistics": {
+      "$ref": "#/$defs/OutputStatisticsOutputDatasetFacet"
+    }
+  }
+}

data/spec/facets/OwnershipDatasetFacet.json

@@ -0,0 +1,45 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-1/OwnershipDatasetFacet.json",
+  "$defs": {
+    "OwnershipDatasetFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/DatasetFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "owners": {
+              "description": "The owners of the dataset.",
+              "type": "array",
+              "items": {
+                "type": "object",
+                "properties": {
+                  "name": {
+                    "description": "the identifier of the owner of the Dataset. It is recommended to define this as a URN. For example application:foo, user:jdoe, team:data",
+                    "type": "string",
+                    "example": "application:app_name"
+                  },
+                  "type": {
+                    "description": "The type of ownership (optional)",
+                    "type": "string",
+                    "example": "MAINTAINER"
+                  }
+                },
+                "required": ["name"]
+              }
+            }
+          }
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "ownership": {
+      "$ref": "#/$defs/OwnershipDatasetFacet"
+    }
+  }
+}

data/spec/facets/OwnershipJobFacet.json

@@ -0,0 +1,45 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-1/OwnershipJobFacet.json",
+  "$defs": {
+    "OwnershipJobFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/JobFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "owners": {
+              "description": "The owners of the job.",
+              "type": "array",
+              "items": {
+                "type": "object",
+                "properties": {
+                  "name": {
+                    "description": "the identifier of the owner of the Job. It is recommended to define this as a URN. For example application:foo, user:jdoe, team:data",
+                    "type": "string",
+                    "example": "application:app_name"
+                  },
+                  "type": {
+                    "description": "The type of ownership (optional)",
+                    "type": "string",
+                    "example": "MAINTAINER"
+                  }
+                },
+                "required": ["name"]
+              }
+            }
+          }
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "ownership": {
+      "$ref": "#/$defs/OwnershipJobFacet"
+    }
+  }
+}

data/spec/facets/ParentRunFacet.json

@@ -0,0 +1,54 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-1/ParentRunFacet.json",
+  "$defs": {
+    "ParentRunFacet": {
+      "description": "the id of the parent run and job, iff this run was spawn from an other run (for example, the Dag run scheduling its tasks)",
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "run": {
+              "type": "object",
+              "properties": {
+                "runId": {
+                  "description": "The globally unique ID of the run associated with the job.",
+                  "type": "string",
+                  "format": "uuid"
+                }
+              },
+              "required": ["runId"]
+            },
+            "job": {
+              "type": "object",
+              "properties": {
+                "namespace": {
+                  "description": "The namespace containing that job",
+                  "type": "string",
+                  "example": "my-scheduler-namespace"
+                },
+                "name": {
+                  "description": "The unique name for that job within that namespace",
+                  "type": "string",
+                  "example": "myjob.mytask"
+                }
+              },
+              "required": ["namespace", "name"]
+            }
+          },
+          "required": ["run", "job"]
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "parent": {
+      "$ref": "#/$defs/ParentRunFacet"
+    }
+  }
+}

data/spec/facets/ProcessingEngineRunFacet.json

@@ -0,0 +1,41 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-1-1/ProcessingEngineRunFacet.json",
+  "$defs": {
+    "ProcessingEngineRunFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "version": {
+              "description": "Processing engine version. Might be Airflow or Spark version.",
+              "type": "string",
+              "example": "2.5.0"
+            },
+            "name": {
+              "description": "Processing engine name, e.g. Airflow or Spark",
+              "type": "string",
+              "example": "Airflow"
+            },
+            "openlineageAdapterVersion": {
+              "description": "OpenLineage adapter package version. Might be e.g. OpenLineage Airflow integration package version",
+              "type": "string",
+              "example": "0.19.0"
+            }
+          },
+          "required": ["version"]
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "processing_engine": {
+      "$ref": "#/$defs/ProcessingEngineRunFacet"
+    }
+  }
+}

data/spec/facets/SQLJobFacet.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-1/SQLJobFacet.json",
+  "$defs": {
+    "SQLJobFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/JobFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "query": {
+              "type": "string",
+              "example": "SELECT * FROM foo"
+            }
+          },
+          "required": ["query"]
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "sql": {
+      "$ref": "#/$defs/SQLJobFacet"
+    }
+  }
+}

data/spec/facets/SchemaDatasetFacet.json

@@ -0,0 +1,59 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-1-1/SchemaDatasetFacet.json",
+  "$defs": {
+    "SchemaDatasetFacetFields": {
+      "type": "object",
+      "properties": {
+        "name": {
+          "description": "The name of the field.",
+          "type": "string",
+          "example": "column1"
+        },
+        "type": {
+          "description": "The type of the field.",
+          "type": "string",
+          "example": "VARCHAR|INT|..."
+        },
+        "description": {
+          "description": "The description of the field.",
+          "type": "string"
+        },
+        "fields": {
+          "description": "Nested struct fields.",
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/SchemaDatasetFacetFields"
+          }
+        }
+      },
+      "required": ["name"]
+    },
+    "SchemaDatasetFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/DatasetFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "fields": {
+              "description": "The fields of the data source.",
+              "type": "array",
+              "items": {
+                "$ref": "#/$defs/SchemaDatasetFacetFields"
+              }
+            }
+          }
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "schema": {
+      "$ref": "#/$defs/SchemaDatasetFacet"
+    }
+  }
+}

data/spec/facets/SourceCodeJobFacet.json

@@ -0,0 +1,34 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-1/SourceCodeJobFacet.json",
+  "$defs": {
+    "SourceCodeJobFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/JobFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "language": {
+              "description": "Language in which source code of this job was written.",
+              "type": "string"
+            },
+            "sourceCode": {
+              "description": "Source code of this job.",
+              "type": "string"
+            }
+          },
+          "required": ["language", "sourceCode"]
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "sourceCode": {
+      "$ref": "#/$defs/SourceCodeJobFacet"
+    }
+  }
+}

data/spec/facets/SourceCodeLocationJobFacet.json

@@ -0,0 +1,60 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-1/SourceCodeLocationJobFacet.json",
+  "$defs": {
+    "SourceCodeLocationJobFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/JobFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "type": {
+              "type": "string",
+              "description": "the source control system",
+              "example": "git|svn"
+            },
+            "url": {
+              "type": "string",
+              "description": "the full http URL to locate the file",
+              "format": "uri",
+              "example": "https://github.com/MarquezProject/marquez-airflow-quickstart/blob/693e35482bc2e526ced2b5f9f76ef83dec6ec691/dags/dummy_example.py"
+            },
+            "repoUrl": {
+              "type": "string",
+              "description": "the URL to the repository",
+              "example": "git@github.com:{org}/{repo}.git or https://github.com/{org}/{repo}.git|svn://<your_ip>/<repository_name>"
+            },
+            "path": {
+              "type": "string",
+              "description": "the path in the repo containing the source files",
+              "example": "path/to/my/dags"
+            },
+            "version": {
+              "type": "string",
+              "description": "the current version deployed (not a branch name, the actual unique version)",
+              "example": "git: the git sha | Svn: the revision number"
+            },
+            "tag": {
+              "type": "string",
+              "description": "optional tag name"
+            },
+            "branch": {
+              "type": "string",
+              "description": "optional branch name"
+            }
+          },
+          "required": ["type", "url"]
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "sourceCodeLocation": {
+      "$ref": "#/$defs/SourceCodeLocationJobFacet"
+    }
+  }
+}

data/spec/facets/StorageDatasetFacet.json

@@ -0,0 +1,35 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-1/StorageDatasetFacet.json",
+  "$defs": {
+    "StorageDatasetFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/DatasetFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "storageLayer": {
+              "description": "Storage layer provider with allowed values: iceberg, delta.",
+              "type": "string"
+            },
+            "fileFormat": {
+              "description": "File format with allowed values: parquet, orc, avro, json, csv, text, xml.",
+              "type": "string"
+            }
+          },
+          "additionalProperties": true,
+          "required": ["storageLayer"]
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "storage": {
+      "$ref": "#/$defs/StorageDatasetFacet"
+    }
+  }
+}

data/spec/facets/SymlinksDatasetFacet.json

@@ -0,0 +1,47 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-1/SymlinksDatasetFacet.json",
+  "$defs": {
+    "SymlinksDatasetFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/DatasetFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "identifiers": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "properties": {
+                  "namespace": {
+                    "type": "string",
+                    "description": "The dataset namespace"
+                  },
+                  "name": {
+                    "type": "string",
+                    "description": "The dataset name"
+                  },
+                  "type": {
+                    "type": "string",
+                    "description": "Identifier type",
+                    "example": "table"
+                  }
+                },
+                "required": ["namespace", "name", "type"]
+              }
+            }
+          }
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "symlinks": {
+      "$ref": "#/$defs/SymlinksDatasetFacet"
+    }
+  }
+}

data/spec/registry/core/registry.json

@@ -0,0 +1,31 @@
+{
+  "producer": {
+    "root_doc_url": "https://github.com/OpenLineage/OpenLineage/tree/main/spec",
+    "produced_facets": [
+      "ol:core:ColumnLineageDatasetFacet.json",
+      "ol:core:DataQualityAssertionsDatasetFacet.json",
+      "ol:core:DataQualityMetricsInputDatasetFacet.json",
+      "ol:core:DatasetVersionDatasetFacet.json",
+      "ol:core:DatasourceDatasetFacet.json",
+      "ol:core:DocumentationDatasetFacet.json",
+      "ol:core:DocumentationJobFacet.json",
+      "ol:core:ErrorMessageRunFacet.json",
+      "ol:core:ExternalQueryRunFacet.json",
+      "ol:core:ExtractionErrorRunFacet.json",
+      "ol:core:JobTypeJobFacet.json",
+      "ol:core:LifecycleStateChangeDatasetFacet.json",
+      "ol:core:NominalTimeRunFacet.json",
+      "ol:core:OutputStatisticsOutputDatasetFacet.json",
+      "ol:core:OwnershipDatasetFacet.json",
+      "ol:core:OwnershipJobFacet.json",
+      "ol:core:ParentRunFacet.json",
+      "ol:core:ProcessingEngineRunFacet.json",
+      "ol:core:SQLJobFacet.json",
+      "ol:core:SchemaDatasetFacet.json",
+      "ol:core:SourceCodeJobFacet.json",
+      "ol:core:SourceCodeLocationJobFacet.json",
+      "ol:core:StorageDatasetFacet.json",
+      "ol:core:SymlinksDatasetFacet.json"
+    ]
+  }
+}