fluent-plugin-openlineage 0.1.1 → 0.1.2

data/spec/facets/StorageDatasetFacet.json

@@ -1,35 +0,0 @@
-{
-  "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "$id": "https://openlineage.io/spec/facets/1-0-1/StorageDatasetFacet.json",
-  "$defs": {
-    "StorageDatasetFacet": {
-      "allOf": [
-        {
-          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/DatasetFacet"
-        },
-        {
-          "type": "object",
-          "properties": {
-            "storageLayer": {
-              "description": "Storage layer provider with allowed values: iceberg, delta.",
-              "type": "string"
-            },
-            "fileFormat": {
-              "description": "File format with allowed values: parquet, orc, avro, json, csv, text, xml.",
-              "type": "string"
-            }
-          },
-          "additionalProperties": true,
-          "required": ["storageLayer"]
-        }
-      ],
-      "type": "object"
-    }
-  },
-  "type": "object",
-  "properties": {
-    "storage": {
-      "$ref": "#/$defs/StorageDatasetFacet"
-    }
-  }
-}

data/spec/facets/SymlinksDatasetFacet.json

@@ -1,47 +0,0 @@
-{
-  "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "$id": "https://openlineage.io/spec/facets/1-0-1/SymlinksDatasetFacet.json",
-  "$defs": {
-    "SymlinksDatasetFacet": {
-      "allOf": [
-        {
-          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/DatasetFacet"
-        },
-        {
-          "type": "object",
-          "properties": {
-            "identifiers": {
-              "type": "array",
-              "items": {
-                "type": "object",
-                "properties": {
-                  "namespace": {
-                    "type": "string",
-                    "description": "The dataset namespace"
-                  },
-                  "name": {
-                    "type": "string",
-                    "description": "The dataset name"
-                  },
-                  "type": {
-                    "type": "string",
-                    "description": "Identifier type",
-                    "example": "table"
-                  }
-                },
-                "required": ["namespace", "name", "type"]
-              }
-            }
-          }
-        }
-      ],
-      "type": "object"
-    }
-  },
-  "type": "object",
-  "properties": {
-    "symlinks": {
-      "$ref": "#/$defs/SymlinksDatasetFacet"
-    }
-  }
-}

data/spec/fluent/plugin/test_parser_openlineage.rb

@@ -1,141 +0,0 @@
-require "helper"
-require "lib/fluent/plugin/parser_openlineage.rb"
-
-class OpenlineageParserTest < Test::Unit::TestCase
-  setup do
-    Fluent::Test.setup
-    @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::OpenlineageParser)
-    @parser.configure(
-      'spec_directory' => '../../spec/'
-    )
-  end
-
- test "test event with no runId" do
-   ol_event = File.read("spec/events/event_no_run_id.json")
-
-   err = assert_raise Fluent::ParserError do
-     @parser.instance.parse(ol_event)
-   end
-   assert_match(/Openlineage validation failed: (.+) path "\/run": "runId" is a required property/, err.message)
- end
-
-  test "test invalid json" do
-    assert_raise Fluent::ParserError do
-      @parser.instance.parse('{"run": Not a JSON}')
-    end
-  end
-
-  test "event full test" do
-    ol_event = File.read("spec/events/event_full.json")
-    @parser.instance.parse(ol_event) { | time, json |
-       assert_equal("ea041791-68bc-4ae1-bd89-4c8106a157e4", json['run']['runId'])
-       assert_equal(2000, json['outputs'][0]['outputFacets']['outputStatistics']['rowCount'])
-    }
-  end
-
-  test "event simple test" do
-    ol_event = File.read("spec/events/event_simple.json")
-    @parser.instance.parse(ol_event) { | time, json |
-      assert_equal("41fb5137-f0fd-4ee5-ba5c-56f8571d1bd7", json['run']['runId'])
-    }
-  end
-
- test "invalid spec_directory test" do
-   assert_raise Fluent::ParserError do
-     @parser.configure(
-       'spec_directory' => './non-existent-spec/'
-     )
-   end
- end
-
-  test "valid spec_directory without slash" do
-    @parser.configure(
-      'spec_directory' => '../../spec'
-    )
-    ol_event = File.read("spec/events/event_simple.json")
-    @parser.instance.parse(ol_event) { | time, json |
-      assert_equal("41fb5137-f0fd-4ee5-ba5c-56f8571d1bd7", json['run']['runId'])
-    }
-  end
-
-  test "run facet validation" do
-    ol_event = File.read("spec/events/event_invalid_run_facet.json")
-    err = assert_raise Fluent::ParserError do
-      @parser.instance.parse(ol_event)
-    end
-    assert_match(/Openlineage validation failed: (.+) path "\/run\/facets/, err.message)
-  end
-
-  test "run facet validation turned off" do
-    ol_event = File.read("spec/events/event_invalid_run_facet.json")
-    @parser.configure(
-      'spec_directory' => '../../spec/',
-      'validate_run_facets' => false,
-    )
-    @parser.instance.parse(ol_event) { | time, json |
-      assert_equal("41fb5137-f0fd-4ee5-ba5c-56f8571d1bd7", json['run']['runId'])
-    }
-  end
-
-  test "job facet validation" do
-    ol_event = File.read("spec/events/event_invalid_job_facet.json")
-    err = assert_raise Fluent::ParserError do
-      @parser.instance.parse(ol_event)
-    end
-    assert_match(/Openlineage validation failed: (.+) path "\/job\/facets\/ownership/, err.message)
-  end
-
-  test "job facet validation turned off" do
-    ol_event = File.read("spec/events/event_invalid_job_facet.json")
-    @parser.configure(
-      'spec_directory' => '../../spec/',
-      'validate_job_facets' => false,
-    )
-    @parser.instance.parse(ol_event) { | time, json |
-      assert_equal("41fb5137-f0fd-4ee5-ba5c-56f8571d1bd7", json['run']['runId'])
-    }
-  end
-
-  test "dataset facet validation" do
-    ol_event = File.read("spec/events/event_invalid_dataset_facet.json")
-    @parser.configure(
-      'spec_directory' => '../../spec/',
-      'validate_dataset_facets' => true,
-    )
-    err = assert_raise Fluent::ParserError do
-      @parser.instance.parse(ol_event)
-    end
-    assert_match(/Openlineage validation failed: (.+) path "\/outputs\/0\/facets\/ownership\/owners/, err.message)
-  end
-
-  test "input dataset facet validation" do
-    ol_event = File.read("spec/events/event_invalid_input_dataset_facet.json")
-     @parser.configure(
-      'json_parser' => 'yajl',
-      'spec_directory' => '../../spec/',
-      'validate_input_dataset_facets' => true,
-    )
-    err = assert_raise Fluent::ParserError do
-      @parser.instance.parse(ol_event)
-    end
-    assert_match(/Openlineage validation failed: (.+) "columnMetrics" is a required property/, err.message)
-  end
-
-  test "output dataset facet validation" do
-    ol_event = File.read("spec/events/event_invalid_output_dataset_facet.json")
-    @parser.configure(
-      'spec_directory' => '../../spec/',
-      'validate_output_dataset_facets' => true,
-    )
-    err = assert_raise Fluent::ParserError do
-      @parser.instance.parse(ol_event)
-    end
-    assert_match(/Openlineage validation failed: (.+) path \"\/outputs\/0\/outputFacets\/outputStatistics\/rowCount/, err.message)
-  end
-
-  private
-
-  def create_driver(conf)
-    Fluent::Test::Driver::Parser.new(Fluent::Plugin::OpenlineageParser).configure(conf)
-  end
-end

data/spec/registry/core/registry.json

@@ -1,31 +0,0 @@
-{
-  "producer": {
-    "root_doc_url": "https://github.com/OpenLineage/OpenLineage/tree/main/spec",
-    "produced_facets": [
-      "ol:core:ColumnLineageDatasetFacet.json",
-      "ol:core:DataQualityAssertionsDatasetFacet.json",
-      "ol:core:DataQualityMetricsInputDatasetFacet.json",
-      "ol:core:DatasetVersionDatasetFacet.json",
-      "ol:core:DatasourceDatasetFacet.json",
-      "ol:core:DocumentationDatasetFacet.json",
-      "ol:core:DocumentationJobFacet.json",
-      "ol:core:ErrorMessageRunFacet.json",
-      "ol:core:ExternalQueryRunFacet.json",
-      "ol:core:ExtractionErrorRunFacet.json",
-      "ol:core:JobTypeJobFacet.json",
-      "ol:core:LifecycleStateChangeDatasetFacet.json",
-      "ol:core:NominalTimeRunFacet.json",
-      "ol:core:OutputStatisticsOutputDatasetFacet.json",
-      "ol:core:OwnershipDatasetFacet.json",
-      "ol:core:OwnershipJobFacet.json",
-      "ol:core:ParentRunFacet.json",
-      "ol:core:ProcessingEngineRunFacet.json",
-      "ol:core:SQLJobFacet.json",
-      "ol:core:SchemaDatasetFacet.json",
-      "ol:core:SourceCodeJobFacet.json",
-      "ol:core:SourceCodeLocationJobFacet.json",
-      "ol:core:StorageDatasetFacet.json",
-      "ol:core:SymlinksDatasetFacet.json"
-    ]
-  }
-}

data/spec/registry/gcp/facets/GcpCommonJobFacet.json

@@ -1,43 +0,0 @@
-{
-  "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "$id": "https://openlineage.io/spec/facets/1-0-0/GcpCommonJobFacet.json",
-  "$defs": {
-    "GcpCommonJobFacet": {
-      "allOf": [
-        {
-          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/JobFacet"
-        },
-        {
-          "type": "object",
-          "properties": {
-            "displayName": {
-              "description": "The name of the job to be used on UI",
-              "type": "string"
-            },
-            "origin": {
-              "type": "object",
-              "properties": {
-                "sourceType": {
-                  "description": "Type of the source. Possible values can be found in GCP documentation (https://cloud.google.com/data-catalog/docs/reference/data-lineage/rest/v1/projects.locations.processes#SourceType) ",
-                  "type": "string"
-                },
-                "name": {
-                  "description": "If the sourceType isn't CUSTOM, the value of this field should be a GCP resource name of the system, which reports lineage. The project and location parts of the resource name must match the project and location of the lineage resource being created. More details in GCP documentation https://cloud.google.com/data-catalog/docs/reference/data-lineage/rest/v1/projects.locations.processes#origin",
-                  "type": "string"
-                }
-              }
-            }
-          },
-          "additionalProperties": true
-        }
-      ],
-      "type": "object"
-    }
-  },
-  "type": "object",
-  "properties": {
-    "gcp_common": {
-      "$ref": "#/$defs/GcpCommonJobFacet"
-    }
-  }
-}

data/spec/registry/gcp/registry.json

@@ -1,6 +0,0 @@
-{
-  "consumer": {
-    "root_doc_URL": "https://cloud.google.com/data-catalog/docs/reference/data-lineage/rpc/google.cloud.datacatalog.lineage.v1#google.cloud.datacatalog.lineage.v1.Lineage.ProcessOpenLineageRunEvent",
-    "produced_facets": ["ol:gcp:GcpCommonJobFacet.json"]
-  }
-}

data/spec/spec_helper.rb

@@ -1,8 +0,0 @@
-$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
-require "test-unit"
-require "fluent/test"
-require "fluent/test/driver/parser"
-require "fluent/test/helpers"
-
-Test::Unit::TestCase.include(Fluent::Test::Helpers)
-Test::Unit::TestCase.extend(Fluent::Test::Helpers)

data/spec/tests/ColumnLineageDatasetFacet/1.json

@@ -1,172 +0,0 @@
-{
-  "columnLineage": {
-    "fields": {
-      "NAME": {
-        "inputFields": [
-          {
-            "namespace": "SnowflakeOpenLineage",
-            "name": "CUSTOMERS",
-            "field": "NAME",
-            "transformations": [
-              {
-                "type": "DIRECT",
-                "subtype": "IDENTITY",
-                "masking": false
-              }
-            ]
-          },
-          {
-            "namespace": "SnowflakeOpenLineage",
-            "name": "CUSTOMERS",
-            "field": "ID",
-            "transformations": [
-              {
-                "type": "INDIRECT",
-                "subtype": "JOIN",
-                "description": "ON (DISCOUNTS.CUSTOMERS_ID=CUSTOMERS.ID)"
-              }
-            ]
-          },
-          {
-            "namespace": "SnowflakeOpenLineage",
-            "name": "DISCOUNTS",
-            "field": "CUSTOMERS_ID",
-            "transformations": [
-              {
-                "type": "INDIRECT",
-                "subtype": "JOIN",
-                "description": "ON (DISCOUNTS.CUSTOMERS_ID=CUSTOMERS.ID)"
-              }
-            ]
-          }
-        ],
-        "transformationDescription": "SELECT NAME",
-        "transformationType": "IDENTITY"
-      },
-      "AMOUNT_OFF": {
-        "inputFields": [
-          {
-            "namespace": "SnowflakeOpenLineage",
-            "name": "DISCOUNTS",
-            "field": "AMOUNT_OFF",
-            "transformations": [
-              {
-                "type": "DIRECT",
-                "subtype": "IDENTITY",
-                "masking": false
-              }
-            ]
-          },
-          {
-            "namespace": "SnowflakeOpenLineage",
-            "name": "CUSTOMERS",
-            "field": "ID",
-            "transformations": [
-              {
-                "type": "INDIRECT",
-                "subtype": "JOIN",
-                "description": "ON (DISCOUNTS.CUSTOMERS_ID=CUSTOMERS.ID)"
-              }
-            ]
-          },
-          {
-            "namespace": "SnowflakeOpenLineage",
-            "name": "DISCOUNTS",
-            "field": "CUSTOMERS_ID",
-            "transformations": [
-              {
-                "type": "INDIRECT",
-                "subtype": "JOIN",
-                "description": "ON (DISCOUNTS.CUSTOMERS_ID=CUSTOMERS.ID)"
-              }
-            ]
-          }
-        ],
-        "transformationType": "IDENTITY"
-      },
-      "STARTS_AT": {
-        "inputFields": [
-          {
-            "namespace": "SnowflakeOpenLineage",
-            "name": "DISCOUNTS",
-            "field": "STARTS_AT",
-            "transformations": [
-              {
-                "type": "DIRECT",
-                "subtype": "IDENTITY",
-                "masking": false
-              }
-            ]
-          },
-          {
-            "namespace": "SnowflakeOpenLineage",
-            "name": "CUSTOMERS",
-            "field": "ID",
-            "transformations": [
-              {
-                "type": "INDIRECT",
-                "subtype": "JOIN",
-                "description": "ON (DISCOUNTS.CUSTOMERS_ID=CUSTOMERS.ID)"
-              }
-            ]
-          },
-          {
-            "namespace": "SnowflakeOpenLineage",
-            "name": "DISCOUNTS",
-            "field": "CUSTOMERS_ID",
-            "transformations": [
-              {
-                "type": "INDIRECT",
-                "subtype": "JOIN",
-                "description": "ON (DISCOUNTS.CUSTOMERS_ID=CUSTOMERS.ID)"
-              }
-            ]
-          }
-        ],
-        "transformationType": "IDENTITY"
-      },
-      "ENDS_AT": {
-        "inputFields": [
-          {
-            "namespace": "SnowflakeOpenLineage",
-            "name": "DISCOUNTS",
-            "field": "ENDS_AT",
-            "transformations": [
-              {
-                "type": "DIRECT",
-                "subtype": "IDENTITY",
-                "masking": false
-              }
-            ]
-          },
-          {
-            "namespace": "SnowflakeOpenLineage",
-            "name": "CUSTOMERS",
-            "field": "ID",
-            "transformations": [
-              {
-                "type": "INDIRECT",
-                "subtype": "JOIN",
-                "description": "ON (DISCOUNTS.CUSTOMERS_ID=CUSTOMERS.ID)"
-              }
-            ]
-          },
-          {
-            "namespace": "SnowflakeOpenLineage",
-            "name": "DISCOUNTS",
-            "field": "CUSTOMERS_ID",
-            "transformations": [
-              {
-                "type": "INDIRECT",
-                "subtype": "JOIN",
-                "description": "ON (DISCOUNTS.CUSTOMERS_ID=CUSTOMERS.ID)"
-              }
-            ]
-          }
-        ]
-      }
-    },
-    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
-    "_schemaURL": "https://openlineage.io/spec/facets/1-1-0/ColumnLineageDatasetFacet.json"
-  }
-}

data/spec/tests/DataQualityAssertionsDatasetFacet/1.json

@@ -1,58 +0,0 @@
-{
-  "dataQualityAssertions": {
-    "assertions": [
-      {
-        "assertion": "accepted_values",
-        "column": "status",
-        "success": true
-      },
-      {
-        "assertion": "not_null",
-        "column": "amount",
-        "success": true
-      },
-      {
-        "assertion": "not_null",
-        "column": "bank_transfer_amount",
-        "success": true
-      },
-      {
-        "assertion": "not_null",
-        "column": "coupon_amount",
-        "success": true
-      },
-      {
-        "assertion": "not_null",
-        "column": "credit_card_amount",
-        "success": true
-      },
-      {
-        "assertion": "not_null",
-        "column": "customer_id",
-        "success": true
-      },
-      {
-        "assertion": "not_null",
-        "column": "gift_card_amount",
-        "success": true
-      },
-      {
-        "assertion": "not_null",
-        "column": "order_id",
-        "success": true
-      },
-      {
-        "assertion": "relationships",
-        "column": "customer_id",
-        "success": true
-      },
-      {
-        "assertion": "unique",
-        "column": "order_id",
-        "success": true
-      }
-    ],
-    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
-    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DataQualityAssertionsDatasetFacet.json"
-  }
-}

data/spec/tests/DataQualityMetricsInputDatasetFacet/1.json

@@ -1,23 +0,0 @@
-{
-  "dataQualityMetrics": {
-    "columnMetrics": {
-      "amount": {
-        "count": 10,
-        "max": 1500,
-        "min": 100,
-        "sum": 6600
-      },
-      "counterparty_id": {
-        "nullCount": 0
-      },
-      "user_id": {
-        "distinctCount": 3,
-        "nullCount": 0
-      }
-    },
-    "rowCount": 10,
-    "fileCount": 5,
-    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
-    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DataQualityMetricsInputDatasetFacet.json"
-  }
-}

data/spec/tests/DatasetVersionDatasetFacet/1.json

@@ -1,7 +0,0 @@
-{
-  "version": {
-    "datasetVersion": "2",
-    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
-    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DatasetVersionDatasetFacet.json"
-  }
-}

data/spec/tests/DatasourceDatasetFacet/1.json

@@ -1,7 +0,0 @@
-{
-  "version": {
-    "datasetVersion": "2",
-    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
-    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DatasetVersionDatasetFacet.json"
-  }
-}

data/spec/tests/DocumentationDatasetFacet/1.json

@@ -1,7 +0,0 @@
-{
-  "documentation": {
-    "description": "Some dataset description",
-    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
-    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DocumentationDatasetFacet.json"
-  }
-}

data/spec/tests/DocumentationJobFacet/1.json

@@ -1,7 +0,0 @@
-{
-  "documentation": {
-    "description": "Determines the popular day of week orders are placed.",
-    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
-    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DocumentationDatasetFacet.json"
-  }
-}

data/spec/tests/ErrorMessageRunFacet/1.json

@@ -1,9 +0,0 @@
-{
-  "errorMessage": {
-    "message": "failed",
-    "programmingLanguage": "JAVA",
-    "stackTrace": "<stack_trace>",
-    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
-    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/ErrorMessageRunFacet.json"
-  }
-}

data/spec/tests/ExternalQueryRunFacet/1.json

@@ -1,8 +0,0 @@
-{
-  "externalQuery": {
-    "_producer": "https://github.com/OpenLineage/OpenLineage/tree/0.0.1/integration/airflow",
-    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/ExternalQueryRunFacet.json",
-    "externalQueryId": "bigquery_1690280798993890_f4ade238a8121576708e1892f69e67fe",
-    "source": "bigquery"
-  }
-}

data/spec/tests/ExtractionErrorRunFacet/1.json

@@ -1,15 +0,0 @@
-{
-  "extractionError": {
-    "totalTasks": 1,
-    "failedTasks": 1,
-    "errors": [
-      {
-        "errorMessage": "Expected TABLE, VIEW, INDEX, ROLE, SCHEMA, FUNCTION, STAGE or SEQUENCE after DROP, found: POLICY at Line: 1, Column 6",
-        "task": "DROP POLICY IF EXISTS name ON table_name",
-        "taskNumber": 0
-      }
-    ],
-    "_producer": "https://github.com/OpenLineage/OpenLineage/tree/0.0.1/integration/dbt",
-    "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/ExtractionErrorRunFacet.json"
-  }
-}

data/spec/tests/JobTypeJobFacet/1.json

@@ -1,9 +0,0 @@
-{
-  "jobType": {
-    "processingType": "BATCH",
-    "integration": "SPARK",
-    "jobType": "QUERY",
-    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
-    "_schemaURL": "https://openlineage.io/spec/facets/2-0-2/JobTypeJobFacet.json"
-  }
-}

data/spec/tests/LifecycleStateChangeDatasetFacet/1.json

@@ -1,11 +0,0 @@
-{
-  "lifecycleStateChange": {
-    "lifecycleStateChange": "RENAME",
-    "previousIdentifier": {
-      "name": "/tmp/alter_test/alter_table_test",
-      "namespace": "file"
-    },
-    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
-    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DatasetVersionDatasetFacet.json"
-  }
-}