fluent-plugin-openlineage 0.1.2 → 0.1.3

data/spec/facets/StorageDatasetFacet.json

@@ -0,0 +1,35 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-1/StorageDatasetFacet.json",
+  "$defs": {
+    "StorageDatasetFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/DatasetFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "storageLayer": {
+              "description": "Storage layer provider with allowed values: iceberg, delta.",
+              "type": "string"
+            },
+            "fileFormat": {
+              "description": "File format with allowed values: parquet, orc, avro, json, csv, text, xml.",
+              "type": "string"
+            }
+          },
+          "additionalProperties": true,
+          "required": ["storageLayer"]
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "storage": {
+      "$ref": "#/$defs/StorageDatasetFacet"
+    }
+  }
+}

data/spec/facets/SymlinksDatasetFacet.json

@@ -0,0 +1,47 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-1/SymlinksDatasetFacet.json",
+  "$defs": {
+    "SymlinksDatasetFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/DatasetFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "identifiers": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "properties": {
+                  "namespace": {
+                    "type": "string",
+                    "description": "The dataset namespace"
+                  },
+                  "name": {
+                    "type": "string",
+                    "description": "The dataset name"
+                  },
+                  "type": {
+                    "type": "string",
+                    "description": "Identifier type",
+                    "example": "table"
+                  }
+                },
+                "required": ["namespace", "name", "type"]
+              }
+            }
+          }
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "symlinks": {
+      "$ref": "#/$defs/SymlinksDatasetFacet"
+    }
+  }
+}

data/spec/fluent/plugin/test_parser_openlineage.rb

@@ -0,0 +1,141 @@
+require "helper"
+require "lib/fluent/plugin/parser_openlineage.rb"
+
+class OpenlineageParserTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+    @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::OpenlineageParser)
+    @parser.configure(
+      'spec_directory' => '../../spec/'
+    )
+  end
+
+ test "test event with no runId" do
+   ol_event = File.read("spec/events/event_no_run_id.json")
+
+   err = assert_raise Fluent::ParserError do
+     @parser.instance.parse(ol_event)
+   end
+   assert_match(/Openlineage validation failed: (.+) path "\/run": "runId" is a required property/, err.message)
+ end
+
+  test "test invalid json" do
+    assert_raise Fluent::ParserError do
+      @parser.instance.parse('{"run": Not a JSON}')
+    end
+  end
+
+  test "event full test" do
+    ol_event = File.read("spec/events/event_full.json")
+    @parser.instance.parse(ol_event) { | time, json |
+       assert_equal("ea041791-68bc-4ae1-bd89-4c8106a157e4", json['run']['runId'])
+       assert_equal(2000, json['outputs'][0]['outputFacets']['outputStatistics']['rowCount'])
+    }
+  end
+
+  test "event simple test" do
+    ol_event = File.read("spec/events/event_simple.json")
+    @parser.instance.parse(ol_event) { | time, json |
+      assert_equal("41fb5137-f0fd-4ee5-ba5c-56f8571d1bd7", json['run']['runId'])
+    }
+  end
+
+ test "invalid spec_directory test" do
+   assert_raise Fluent::ParserError do
+     @parser.configure(
+       'spec_directory' => './non-existent-spec/'
+     )
+   end
+ end
+
+  test "valid spec_directory without slash" do
+    @parser.configure(
+      'spec_directory' => '../../spec'
+    )
+    ol_event = File.read("spec/events/event_simple.json")
+    @parser.instance.parse(ol_event) { | time, json |
+      assert_equal("41fb5137-f0fd-4ee5-ba5c-56f8571d1bd7", json['run']['runId'])
+    }
+  end
+
+  test "run facet validation" do
+    ol_event = File.read("spec/events/event_invalid_run_facet.json")
+    err = assert_raise Fluent::ParserError do
+      @parser.instance.parse(ol_event)
+    end
+    assert_match(/Openlineage validation failed: (.+) path "\/run\/facets/, err.message)
+  end
+
+  test "run facet validation turned off" do
+    ol_event = File.read("spec/events/event_invalid_run_facet.json")
+    @parser.configure(
+      'spec_directory' => '../../spec/',
+      'validate_run_facets' => false,
+    )
+    @parser.instance.parse(ol_event) { | time, json |
+      assert_equal("41fb5137-f0fd-4ee5-ba5c-56f8571d1bd7", json['run']['runId'])
+    }
+  end
+
+  test "job facet validation" do
+    ol_event = File.read("spec/events/event_invalid_job_facet.json")
+    err = assert_raise Fluent::ParserError do
+      @parser.instance.parse(ol_event)
+    end
+    assert_match(/Openlineage validation failed: (.+) path "\/job\/facets\/ownership/, err.message)
+  end
+
+  test "job facet validation turned off" do
+    ol_event = File.read("spec/events/event_invalid_job_facet.json")
+    @parser.configure(
+      'spec_directory' => '../../spec/',
+      'validate_job_facets' => false,
+    )
+    @parser.instance.parse(ol_event) { | time, json |
+      assert_equal("41fb5137-f0fd-4ee5-ba5c-56f8571d1bd7", json['run']['runId'])
+    }
+  end
+
+  test "dataset facet validation" do
+    ol_event = File.read("spec/events/event_invalid_dataset_facet.json")
+    @parser.configure(
+      'spec_directory' => '../../spec/',
+      'validate_dataset_facets' => true,
+    )
+    err = assert_raise Fluent::ParserError do
+      @parser.instance.parse(ol_event)
+    end
+    assert_match(/Openlineage validation failed: (.+) path "\/outputs\/0\/facets\/ownership\/owners/, err.message)
+  end
+
+  test "input dataset facet validation" do
+    ol_event = File.read("spec/events/event_invalid_input_dataset_facet.json")
+     @parser.configure(
+      'json_parser' => 'yajl',
+      'spec_directory' => '../../spec/',
+      'validate_input_dataset_facets' => true,
+    )
+    err = assert_raise Fluent::ParserError do
+      @parser.instance.parse(ol_event)
+    end
+    assert_match(/Openlineage validation failed: (.+) "columnMetrics" is a required property/, err.message)
+  end
+
+  test "output dataset facet validation" do
+    ol_event = File.read("spec/events/event_invalid_output_dataset_facet.json")
+    @parser.configure(
+      'spec_directory' => '../../spec/',
+      'validate_output_dataset_facets' => true,
+    )
+    err = assert_raise Fluent::ParserError do
+      @parser.instance.parse(ol_event)
+    end
+    assert_match(/Openlineage validation failed: (.+) path \"\/outputs\/0\/outputFacets\/outputStatistics\/rowCount/, err.message)
+  end
+
+  private
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Parser.new(Fluent::Plugin::OpenlineageParser).configure(conf)
+  end
+end

data/spec/registry/core/registry.json

@@ -0,0 +1,31 @@
+{
+  "producer": {
+    "root_doc_url": "https://github.com/OpenLineage/OpenLineage/tree/main/spec",
+    "produced_facets": [
+      "ol:core:ColumnLineageDatasetFacet.json",
+      "ol:core:DataQualityAssertionsDatasetFacet.json",
+      "ol:core:DataQualityMetricsInputDatasetFacet.json",
+      "ol:core:DatasetVersionDatasetFacet.json",
+      "ol:core:DatasourceDatasetFacet.json",
+      "ol:core:DocumentationDatasetFacet.json",
+      "ol:core:DocumentationJobFacet.json",
+      "ol:core:ErrorMessageRunFacet.json",
+      "ol:core:ExternalQueryRunFacet.json",
+      "ol:core:ExtractionErrorRunFacet.json",
+      "ol:core:JobTypeJobFacet.json",
+      "ol:core:LifecycleStateChangeDatasetFacet.json",
+      "ol:core:NominalTimeRunFacet.json",
+      "ol:core:OutputStatisticsOutputDatasetFacet.json",
+      "ol:core:OwnershipDatasetFacet.json",
+      "ol:core:OwnershipJobFacet.json",
+      "ol:core:ParentRunFacet.json",
+      "ol:core:ProcessingEngineRunFacet.json",
+      "ol:core:SQLJobFacet.json",
+      "ol:core:SchemaDatasetFacet.json",
+      "ol:core:SourceCodeJobFacet.json",
+      "ol:core:SourceCodeLocationJobFacet.json",
+      "ol:core:StorageDatasetFacet.json",
+      "ol:core:SymlinksDatasetFacet.json"
+    ]
+  }
+}

data/spec/registry/gcp/facets/GcpCommonJobFacet.json

@@ -0,0 +1,43 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-0/GcpCommonJobFacet.json",
+  "$defs": {
+    "GcpCommonJobFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/JobFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "displayName": {
+              "description": "The name of the job to be used on UI",
+              "type": "string"
+            },
+            "origin": {
+              "type": "object",
+              "properties": {
+                "sourceType": {
+                  "description": "Type of the source. Possible values can be found in GCP documentation (https://cloud.google.com/data-catalog/docs/reference/data-lineage/rest/v1/projects.locations.processes#SourceType) ",
+                  "type": "string"
+                },
+                "name": {
+                  "description": "If the sourceType isn't CUSTOM, the value of this field should be a GCP resource name of the system, which reports lineage. The project and location parts of the resource name must match the project and location of the lineage resource being created. More details in GCP documentation https://cloud.google.com/data-catalog/docs/reference/data-lineage/rest/v1/projects.locations.processes#origin",
+                  "type": "string"
+                }
+              }
+            }
+          },
+          "additionalProperties": true
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "gcp_common": {
+      "$ref": "#/$defs/GcpCommonJobFacet"
+    }
+  }
+}

data/spec/registry/gcp/registry.json

@@ -0,0 +1,6 @@
+{
+  "consumer": {
+    "root_doc_URL": "https://cloud.google.com/data-catalog/docs/reference/data-lineage/rpc/google.cloud.datacatalog.lineage.v1#google.cloud.datacatalog.lineage.v1.Lineage.ProcessOpenLineageRunEvent",
+    "produced_facets": ["ol:gcp:GcpCommonJobFacet.json"]
+  }
+}

data/spec/spec_helper.rb

@@ -0,0 +1,8 @@
+$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
+require "test-unit"
+require "fluent/test"
+require "fluent/test/driver/parser"
+require "fluent/test/helpers"
+
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
+Test::Unit::TestCase.extend(Fluent::Test::Helpers)

data/spec/tests/ColumnLineageDatasetFacet/1.json

@@ -0,0 +1,172 @@
+{
+  "columnLineage": {
+    "fields": {
+      "NAME": {
+        "inputFields": [
+          {
+            "namespace": "SnowflakeOpenLineage",
+            "name": "CUSTOMERS",
+            "field": "NAME",
+            "transformations": [
+              {
+                "type": "DIRECT",
+                "subtype": "IDENTITY",
+                "masking": false
+              }
+            ]
+          },
+          {
+            "namespace": "SnowflakeOpenLineage",
+            "name": "CUSTOMERS",
+            "field": "ID",
+            "transformations": [
+              {
+                "type": "INDIRECT",
+                "subtype": "JOIN",
+                "description": "ON (DISCOUNTS.CUSTOMERS_ID=CUSTOMERS.ID)"
+              }
+            ]
+          },
+          {
+            "namespace": "SnowflakeOpenLineage",
+            "name": "DISCOUNTS",
+            "field": "CUSTOMERS_ID",
+            "transformations": [
+              {
+                "type": "INDIRECT",
+                "subtype": "JOIN",
+                "description": "ON (DISCOUNTS.CUSTOMERS_ID=CUSTOMERS.ID)"
+              }
+            ]
+          }
+        ],
+        "transformationDescription": "SELECT NAME",
+        "transformationType": "IDENTITY"
+      },
+      "AMOUNT_OFF": {
+        "inputFields": [
+          {
+            "namespace": "SnowflakeOpenLineage",
+            "name": "DISCOUNTS",
+            "field": "AMOUNT_OFF",
+            "transformations": [
+              {
+                "type": "DIRECT",
+                "subtype": "IDENTITY",
+                "masking": false
+              }
+            ]
+          },
+          {
+            "namespace": "SnowflakeOpenLineage",
+            "name": "CUSTOMERS",
+            "field": "ID",
+            "transformations": [
+              {
+                "type": "INDIRECT",
+                "subtype": "JOIN",
+                "description": "ON (DISCOUNTS.CUSTOMERS_ID=CUSTOMERS.ID)"
+              }
+            ]
+          },
+          {
+            "namespace": "SnowflakeOpenLineage",
+            "name": "DISCOUNTS",
+            "field": "CUSTOMERS_ID",
+            "transformations": [
+              {
+                "type": "INDIRECT",
+                "subtype": "JOIN",
+                "description": "ON (DISCOUNTS.CUSTOMERS_ID=CUSTOMERS.ID)"
+              }
+            ]
+          }
+        ],
+        "transformationType": "IDENTITY"
+      },
+      "STARTS_AT": {
+        "inputFields": [
+          {
+            "namespace": "SnowflakeOpenLineage",
+            "name": "DISCOUNTS",
+            "field": "STARTS_AT",
+            "transformations": [
+              {
+                "type": "DIRECT",
+                "subtype": "IDENTITY",
+                "masking": false
+              }
+            ]
+          },
+          {
+            "namespace": "SnowflakeOpenLineage",
+            "name": "CUSTOMERS",
+            "field": "ID",
+            "transformations": [
+              {
+                "type": "INDIRECT",
+                "subtype": "JOIN",
+                "description": "ON (DISCOUNTS.CUSTOMERS_ID=CUSTOMERS.ID)"
+              }
+            ]
+          },
+          {
+            "namespace": "SnowflakeOpenLineage",
+            "name": "DISCOUNTS",
+            "field": "CUSTOMERS_ID",
+            "transformations": [
+              {
+                "type": "INDIRECT",
+                "subtype": "JOIN",
+                "description": "ON (DISCOUNTS.CUSTOMERS_ID=CUSTOMERS.ID)"
+              }
+            ]
+          }
+        ],
+        "transformationType": "IDENTITY"
+      },
+      "ENDS_AT": {
+        "inputFields": [
+          {
+            "namespace": "SnowflakeOpenLineage",
+            "name": "DISCOUNTS",
+            "field": "ENDS_AT",
+            "transformations": [
+              {
+                "type": "DIRECT",
+                "subtype": "IDENTITY",
+                "masking": false
+              }
+            ]
+          },
+          {
+            "namespace": "SnowflakeOpenLineage",
+            "name": "CUSTOMERS",
+            "field": "ID",
+            "transformations": [
+              {
+                "type": "INDIRECT",
+                "subtype": "JOIN",
+                "description": "ON (DISCOUNTS.CUSTOMERS_ID=CUSTOMERS.ID)"
+              }
+            ]
+          },
+          {
+            "namespace": "SnowflakeOpenLineage",
+            "name": "DISCOUNTS",
+            "field": "CUSTOMERS_ID",
+            "transformations": [
+              {
+                "type": "INDIRECT",
+                "subtype": "JOIN",
+                "description": "ON (DISCOUNTS.CUSTOMERS_ID=CUSTOMERS.ID)"
+              }
+            ]
+          }
+        ]
+      }
+    },
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-1-0/ColumnLineageDatasetFacet.json"
+  }
+}

data/spec/tests/DataQualityAssertionsDatasetFacet/1.json

@@ -0,0 +1,58 @@
+{
+  "dataQualityAssertions": {
+    "assertions": [
+      {
+        "assertion": "accepted_values",
+        "column": "status",
+        "success": true
+      },
+      {
+        "assertion": "not_null",
+        "column": "amount",
+        "success": true
+      },
+      {
+        "assertion": "not_null",
+        "column": "bank_transfer_amount",
+        "success": true
+      },
+      {
+        "assertion": "not_null",
+        "column": "coupon_amount",
+        "success": true
+      },
+      {
+        "assertion": "not_null",
+        "column": "credit_card_amount",
+        "success": true
+      },
+      {
+        "assertion": "not_null",
+        "column": "customer_id",
+        "success": true
+      },
+      {
+        "assertion": "not_null",
+        "column": "gift_card_amount",
+        "success": true
+      },
+      {
+        "assertion": "not_null",
+        "column": "order_id",
+        "success": true
+      },
+      {
+        "assertion": "relationships",
+        "column": "customer_id",
+        "success": true
+      },
+      {
+        "assertion": "unique",
+        "column": "order_id",
+        "success": true
+      }
+    ],
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DataQualityAssertionsDatasetFacet.json"
+  }
+}

data/spec/tests/DataQualityMetricsInputDatasetFacet/1.json

@@ -0,0 +1,23 @@
+{
+  "dataQualityMetrics": {
+    "columnMetrics": {
+      "amount": {
+        "count": 10,
+        "max": 1500,
+        "min": 100,
+        "sum": 6600
+      },
+      "counterparty_id": {
+        "nullCount": 0
+      },
+      "user_id": {
+        "distinctCount": 3,
+        "nullCount": 0
+      }
+    },
+    "rowCount": 10,
+    "fileCount": 5,
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DataQualityMetricsInputDatasetFacet.json"
+  }
+}

data/spec/tests/DatasetVersionDatasetFacet/1.json

@@ -0,0 +1,7 @@
+{
+  "version": {
+    "datasetVersion": "2",
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DatasetVersionDatasetFacet.json"
+  }
+}

data/spec/tests/DatasourceDatasetFacet/1.json

@@ -0,0 +1,7 @@
+{
+  "version": {
+    "datasetVersion": "2",
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DatasetVersionDatasetFacet.json"
+  }
+}

data/spec/tests/DocumentationDatasetFacet/1.json

@@ -0,0 +1,7 @@
+{
+  "documentation": {
+    "description": "Some dataset description",
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DocumentationDatasetFacet.json"
+  }
+}

data/spec/tests/DocumentationJobFacet/1.json

@@ -0,0 +1,7 @@
+{
+  "documentation": {
+    "description": "Determines the popular day of week orders are placed.",
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DocumentationDatasetFacet.json"
+  }
+}

data/spec/tests/ErrorMessageRunFacet/1.json

@@ -0,0 +1,9 @@
+{
+  "errorMessage": {
+    "message": "failed",
+    "programmingLanguage": "JAVA",
+    "stackTrace": "<stack_trace>",
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/ErrorMessageRunFacet.json"
+  }
+}

data/spec/tests/ExternalQueryRunFacet/1.json

@@ -0,0 +1,8 @@
+{
+  "externalQuery": {
+    "_producer": "https://github.com/OpenLineage/OpenLineage/tree/0.0.1/integration/airflow",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/ExternalQueryRunFacet.json",
+    "externalQueryId": "bigquery_1690280798993890_f4ade238a8121576708e1892f69e67fe",
+    "source": "bigquery"
+  }
+}

data/spec/tests/ExtractionErrorRunFacet/1.json

@@ -0,0 +1,15 @@
+{
+  "extractionError": {
+    "totalTasks": 1,
+    "failedTasks": 1,
+    "errors": [
+      {
+        "errorMessage": "Expected TABLE, VIEW, INDEX, ROLE, SCHEMA, FUNCTION, STAGE or SEQUENCE after DROP, found: POLICY at Line: 1, Column 6",
+        "task": "DROP POLICY IF EXISTS name ON table_name",
+        "taskNumber": 0
+      }
+    ],
+    "_producer": "https://github.com/OpenLineage/OpenLineage/tree/0.0.1/integration/dbt",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/ExtractionErrorRunFacet.json"
+  }
+}

data/spec/tests/JobTypeJobFacet/1.json

@@ -0,0 +1,9 @@
+{
+  "jobType": {
+    "processingType": "BATCH",
+    "integration": "SPARK",
+    "jobType": "QUERY",
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/2-0-2/JobTypeJobFacet.json"
+  }
+}

data/spec/tests/LifecycleStateChangeDatasetFacet/1.json

@@ -0,0 +1,11 @@
+{
+  "lifecycleStateChange": {
+    "lifecycleStateChange": "RENAME",
+    "previousIdentifier": {
+      "name": "/tmp/alter_test/alter_table_test",
+      "namespace": "file"
+    },
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DatasetVersionDatasetFacet.json"
+  }
+}