fluentd-openlineage-parser 0.1.0

data/spec/tests/SourceCodeLocationJobFacet/1.json

@@ -0,0 +1,8 @@
+{
+  "sourceCodeLocation": {
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/SourceCodeLocationJobFacet.json",
+    "type": "git",
+    "url": "http://example.com"
+  }
+}

data/spec/tests/StorageDatasetFacet/1.json

@@ -0,0 +1,8 @@
+{
+  "storage": {
+    "storageLayer": "delta",
+    "fileFormat": "parquet",
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/StorageDatasetFacet.json"
+  }
+}

data/spec/tests/SymlinksDatasetFacet/1.json

@@ -0,0 +1,13 @@
+{
+  "symlinks": {
+    "identifiers": [
+      {
+        "namespace": "/tmp/alter_test",
+        "name": "default.alter_table_test_new",
+        "type": "TABLE"
+      }
+    ],
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/SymlinksDatasetFacet.json"
+  }
+}

data/spec/tests/example_full_event.json

@@ -0,0 +1,24 @@
+{
+  "eventTime": "2023-07-17T10:54:22.355067Z",
+  "eventType": "COMPLETE",
+  "inputs": [],
+  "job": {
+    "facets": {
+      "sql": {
+        "query": "SELECT 1",
+        "_producer": "http://123",
+        "_schemaURL": "http://123"
+      }
+    },
+    "name": "dbt",
+    "namespace": "food_delivery"
+  },
+  "outputs": [],
+  "producer": "https://github.com/OpenLineage/OpenLineage/tree/0.30.0/integration/airflow",
+  "run": {
+    "facets": {},
+    "runId": "f69a6e9b-9bac-3c9a-9cf6-eacb70ecc9a9"
+  },
+  "dataset": { "namespace": "123", "name": "1" },
+  "schemaURL": "https://openlineage.io/spec/1-0-5/OpenLineage.json#/definitions/RunEvent"
+}

data/test/helper.rb

@@ -0,0 +1,8 @@
+$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
+require "test-unit"
+require "fluent/test"
+require "fluent/test/driver/parser"
+require "fluent/test/helpers"
+
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
+Test::Unit::TestCase.extend(Fluent::Test::Helpers)

data/test/plugin/test_parser_openlineage.rb

@@ -0,0 +1,141 @@
+require "helper"
+require "fluent/plugin/parser_openlineage.rb"
+
+class OpenlineageParserTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+    @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::OpenlineageParser)
+    @parser.configure(
+      'spec_directory' => '../../spec/'
+    )
+  end
+
+ test "test event with no runId" do
+   ol_event = File.read("events/event_no_run_id.json")
+
+   err = assert_raise Fluent::ParserError do
+     @parser.instance.parse(ol_event)
+   end
+   assert_match(/Openlineage validation failed: (.+) path "\/run": "runId" is a required property/, err.message)
+ end
+
+  test "test invalid json" do
+    assert_raise Fluent::ParserError do
+      @parser.instance.parse('{"run": Not a JSON}')
+    end
+  end
+
+  test "event full test" do
+    ol_event = File.read("events/event_full.json")
+    @parser.instance.parse(ol_event) { | time, json |
+       assert_equal("ea041791-68bc-4ae1-bd89-4c8106a157e4", json['run']['runId'])
+       assert_equal(2000, json['outputs'][0]['outputFacets']['outputStatistics']['rowCount'])
+    }
+  end
+
+  test "event simple test" do
+    ol_event = File.read("events/event_simple.json")
+    @parser.instance.parse(ol_event) { | time, json |
+      assert_equal("41fb5137-f0fd-4ee5-ba5c-56f8571d1bd7", json['run']['runId'])
+    }
+  end
+
+ test "invalid spec_directory test" do
+   assert_raise Fluent::ParserError do
+     @parser.configure(
+       'spec_directory' => './non-existent-spec/'
+     )
+   end
+ end
+
+  test "valid spec_directory without slash" do
+    @parser.configure(
+      'spec_directory' => '../../spec'
+    )
+    ol_event = File.read("events/event_simple.json")
+    @parser.instance.parse(ol_event) { | time, json |
+      assert_equal("41fb5137-f0fd-4ee5-ba5c-56f8571d1bd7", json['run']['runId'])
+    }
+  end
+
+  test "run facet validation" do
+    ol_event = File.read("events/event_invalid_run_facet.json")
+    err = assert_raise Fluent::ParserError do
+      @parser.instance.parse(ol_event)
+    end
+    assert_match(/Openlineage validation failed: (.+) path "\/run\/facets/, err.message)
+  end
+
+  test "run facet validation turned off" do
+    ol_event = File.read("events/event_invalid_run_facet.json")
+    @parser.configure(
+      'spec_directory' => '../../spec/',
+      'validate_run_facets' => false,
+    )
+    @parser.instance.parse(ol_event) { | time, json |
+      assert_equal("41fb5137-f0fd-4ee5-ba5c-56f8571d1bd7", json['run']['runId'])
+    }
+  end
+
+  test "job facet validation" do
+    ol_event = File.read("events/event_invalid_job_facet.json")
+    err = assert_raise Fluent::ParserError do
+      @parser.instance.parse(ol_event)
+    end
+    assert_match(/Openlineage validation failed: (.+) path "\/job\/facets\/ownership/, err.message)
+  end
+
+  test "job facet validation turned off" do
+    ol_event = File.read("events/event_invalid_job_facet.json")
+    @parser.configure(
+      'spec_directory' => '../../spec/',
+      'validate_job_facets' => false,
+    )
+    @parser.instance.parse(ol_event) { | time, json |
+      assert_equal("41fb5137-f0fd-4ee5-ba5c-56f8571d1bd7", json['run']['runId'])
+    }
+  end
+
+  test "dataset facet validation" do
+    ol_event = File.read("events/event_invalid_dataset_facet.json")
+    @parser.configure(
+      'spec_directory' => '../../spec/',
+      'validate_dataset_facets' => true,
+    )
+    err = assert_raise Fluent::ParserError do
+      @parser.instance.parse(ol_event)
+    end
+    assert_match(/Openlineage validation failed: (.+) path "\/outputs\/0\/facets\/ownership\/owners/, err.message)
+  end
+
+  test "input dataset facet validation" do
+    ol_event = File.read("events/event_invalid_input_dataset_facet.json")
+     @parser.configure(
+      'json_parser' => 'yajl',
+      'spec_directory' => '../../spec/',
+      'validate_input_dataset_facets' => true,
+    )
+    err = assert_raise Fluent::ParserError do
+      @parser.instance.parse(ol_event)
+    end
+    assert_match(/Openlineage validation failed: (.+) "columnMetrics" is a required property/, err.message)
+  end
+
+  test "output dataset facet validation" do
+    ol_event = File.read("events/event_invalid_output_dataset_facet.json")
+    @parser.configure(
+      'spec_directory' => '../../spec/',
+      'validate_output_dataset_facets' => true,
+    )
+    err = assert_raise Fluent::ParserError do
+      @parser.instance.parse(ol_event)
+    end
+    assert_match(/Openlineage validation failed: (.+) path \"\/outputs\/0\/outputFacets\/outputStatistics\/rowCount/, err.message)
+  end
+
+  private
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Parser.new(Fluent::Plugin::OpenlineageParser).configure(conf)
+  end
+end

metadata

@@ -0,0 +1,298 @@
+--- !ruby/object:Gem::Specification
+name: fluentd-openlineage-parser
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Pawel Leszczynski
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2024-08-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.10
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.10
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: rusty_json_schema
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd parser that validates if JSON is a valid Openlineage event.
+email:
+- leszczynski.pawel@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".idea/.gitignore"
+- ".idea/fluentd.iml"
+- ".idea/misc.xml"
+- ".idea/modules.xml"
+- ".idea/modules/benchmark-memory-0.2.iml"
+- ".idea/modules/bigdecimal-3.1.iml"
+- ".idea/modules/certstore_c-0.1.iml"
+- ".idea/modules/concurrent-ruby-1.3.iml"
+- ".idea/modules/concurrent-ruby-1.31.iml"
+- ".idea/modules/connection_pool-2.4.iml"
+- ".idea/modules/cool.io-1.8.iml"
+- ".idea/modules/drb-2.2.iml"
+- ".idea/modules/drb-2.21.iml"
+- ".idea/modules/ffi-1.17.iml"
+- ".idea/modules/ffi-win32-extensions-1.0.iml"
+- ".idea/modules/fluentd-1.17.iml"
+- ".idea/modules/http_parser.rb-0.8.iml"
+- ".idea/modules/json-2.7.iml"
+- ".idea/modules/json-2.71.iml"
+- ".idea/modules/msgpack-1.7.iml"
+- ".idea/modules/mutex_m-0.2.iml"
+- ".idea/modules/new_gem.iml"
+- ".idea/modules/power_assert-2.0.iml"
+- ".idea/modules/rake-13.2.iml"
+- ".idea/modules/rake-13.21.iml"
+- ".idea/modules/rake-compiler-1.2.iml"
+- ".idea/modules/rusty_json_schema-0.15.iml"
+- ".idea/modules/serverengine-2.3.iml"
+- ".idea/modules/sigdump-0.2.iml"
+- ".idea/modules/specifications.iml"
+- ".idea/modules/specifications1.iml"
+- ".idea/modules/strptime-0.2.iml"
+- ".idea/modules/thermite-0.13.iml"
+- ".idea/modules/webrick-1.8.iml"
+- ".idea/modules/win32-event-0.6.iml"
+- ".idea/modules/win32-ipc-0.7.iml"
+- ".idea/modules/yajl-ruby-1.4.iml"
+- Gemfile
+- README.md
+- Rakefile
+- config/conf/fluent.conf
+- config/test-complete.json
+- config/test-start.json
+- events/event_full.json
+- events/event_invalid_dataset_facet.json
+- events/event_invalid_input_dataset_facet.json
+- events/event_invalid_job_facet.json
+- events/event_invalid_output_dataset_facet.json
+- events/event_invalid_run_facet.json
+- events/event_no_run_id.json
+- events/event_simple.json
+- fluentd-openlineage-parser.gemspec
+- lib/fluent/plugin/fluentd-openlineage-parser.rb
+- spec/Naming.md
+- spec/OpenLineage.json
+- spec/OpenLineage.md
+- spec/OpenLineage.yml
+- spec/OpenLineageModel.svg
+- spec/Versioning.md
+- spec/facets/ColumnLineageDatasetFacet.json
+- spec/facets/ColumnLineageDatasetFacet.md
+- spec/facets/DataQualityAssertionsDatasetFacet.json
+- spec/facets/DataQualityMetricsInputDatasetFacet.json
+- spec/facets/DatasetVersionDatasetFacet.json
+- spec/facets/DatasourceDatasetFacet.json
+- spec/facets/DocumentationDatasetFacet.json
+- spec/facets/DocumentationJobFacet.json
+- spec/facets/ErrorMessageRunFacet.json
+- spec/facets/ExternalQueryRunFacet.json
+- spec/facets/ExternalQueryRunFacet.md
+- spec/facets/ExtractionErrorRunFacet.json
+- spec/facets/JobTypeJobFacet.json
+- spec/facets/LifecycleStateChangeDatasetFacet.json
+- spec/facets/NominalTimeRunFacet.json
+- spec/facets/OutputStatisticsOutputDatasetFacet.json
+- spec/facets/OwnershipDatasetFacet.json
+- spec/facets/OwnershipJobFacet.json
+- spec/facets/ParentRunFacet.json
+- spec/facets/ProcessingEngineRunFacet.json
+- spec/facets/SQLJobFacet.json
+- spec/facets/SchemaDatasetFacet.json
+- spec/facets/SourceCodeJobFacet.json
+- spec/facets/SourceCodeLocationJobFacet.json
+- spec/facets/StorageDatasetFacet.json
+- spec/facets/SymlinksDatasetFacet.json
+- spec/registry/core/registry.json
+- spec/registry/gcp/facets/GcpCommonJobFacet.json
+- spec/registry/gcp/registry.json
+- spec/release.sh
+- spec/tests/ColumnLineageDatasetFacet/1.json
+- spec/tests/DataQualityAssertionsDatasetFacet/1.json
+- spec/tests/DataQualityMetricsInputDatasetFacet/1.json
+- spec/tests/DatasetVersionDatasetFacet/1.json
+- spec/tests/DatasourceDatasetFacet/1.json
+- spec/tests/DocumentationDatasetFacet/1.json
+- spec/tests/DocumentationJobFacet/1.json
+- spec/tests/ErrorMessageRunFacet/1.json
+- spec/tests/ExternalQueryRunFacet/1.json
+- spec/tests/ExtractionErrorRunFacet/1.json
+- spec/tests/JobTypeJobFacet/1.json
+- spec/tests/LifecycleStateChangeDatasetFacet/1.json
+- spec/tests/NominalTimeRunFacet/1.json
+- spec/tests/OutputStatisticsOutputDatasetFacet/1.json
+- spec/tests/OwnershipDatasetFacet/1.json
+- spec/tests/OwnershipJobFacet/1.json
+- spec/tests/ParentRunFacet/1.json
+- spec/tests/ProcessingEngineRunFacet/1.json
+- spec/tests/SQLJobFacet/1.json
+- spec/tests/SchemaDatasetFacet/1.json
+- spec/tests/SourceCodeJobFacet/1.json
+- spec/tests/SourceCodeLocationJobFacet/1.json
+- spec/tests/StorageDatasetFacet/1.json
+- spec/tests/SymlinksDatasetFacet/1.json
+- spec/tests/example_full_event.json
+- test/helper.rb
+- test/plugin/test_parser_openlineage.rb
+homepage: http://openlineage.io
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.16
+signing_key:
+specification_version: 4
+summary: Parser to validate Openlineage events.
+test_files:
+- spec/Naming.md
+- spec/OpenLineage.json
+- spec/OpenLineage.md
+- spec/OpenLineage.yml
+- spec/OpenLineageModel.svg
+- spec/Versioning.md
+- spec/facets/ColumnLineageDatasetFacet.json
+- spec/facets/ColumnLineageDatasetFacet.md
+- spec/facets/DataQualityAssertionsDatasetFacet.json
+- spec/facets/DataQualityMetricsInputDatasetFacet.json
+- spec/facets/DatasetVersionDatasetFacet.json
+- spec/facets/DatasourceDatasetFacet.json
+- spec/facets/DocumentationDatasetFacet.json
+- spec/facets/DocumentationJobFacet.json
+- spec/facets/ErrorMessageRunFacet.json
+- spec/facets/ExternalQueryRunFacet.json
+- spec/facets/ExternalQueryRunFacet.md
+- spec/facets/ExtractionErrorRunFacet.json
+- spec/facets/JobTypeJobFacet.json
+- spec/facets/LifecycleStateChangeDatasetFacet.json
+- spec/facets/NominalTimeRunFacet.json
+- spec/facets/OutputStatisticsOutputDatasetFacet.json
+- spec/facets/OwnershipDatasetFacet.json
+- spec/facets/OwnershipJobFacet.json
+- spec/facets/ParentRunFacet.json
+- spec/facets/ProcessingEngineRunFacet.json
+- spec/facets/SQLJobFacet.json
+- spec/facets/SchemaDatasetFacet.json
+- spec/facets/SourceCodeJobFacet.json
+- spec/facets/SourceCodeLocationJobFacet.json
+- spec/facets/StorageDatasetFacet.json
+- spec/facets/SymlinksDatasetFacet.json
+- spec/registry/core/registry.json
+- spec/registry/gcp/facets/GcpCommonJobFacet.json
+- spec/registry/gcp/registry.json
+- spec/release.sh
+- spec/tests/ColumnLineageDatasetFacet/1.json
+- spec/tests/DataQualityAssertionsDatasetFacet/1.json
+- spec/tests/DataQualityMetricsInputDatasetFacet/1.json
+- spec/tests/DatasetVersionDatasetFacet/1.json
+- spec/tests/DatasourceDatasetFacet/1.json
+- spec/tests/DocumentationDatasetFacet/1.json
+- spec/tests/DocumentationJobFacet/1.json
+- spec/tests/ErrorMessageRunFacet/1.json
+- spec/tests/ExternalQueryRunFacet/1.json
+- spec/tests/ExtractionErrorRunFacet/1.json
+- spec/tests/JobTypeJobFacet/1.json
+- spec/tests/LifecycleStateChangeDatasetFacet/1.json
+- spec/tests/NominalTimeRunFacet/1.json
+- spec/tests/OutputStatisticsOutputDatasetFacet/1.json
+- spec/tests/OwnershipDatasetFacet/1.json
+- spec/tests/OwnershipJobFacet/1.json
+- spec/tests/ParentRunFacet/1.json
+- spec/tests/ProcessingEngineRunFacet/1.json
+- spec/tests/SQLJobFacet/1.json
+- spec/tests/SchemaDatasetFacet/1.json
+- spec/tests/SourceCodeJobFacet/1.json
+- spec/tests/SourceCodeLocationJobFacet/1.json
+- spec/tests/StorageDatasetFacet/1.json
+- spec/tests/SymlinksDatasetFacet/1.json
+- spec/tests/example_full_event.json
+- test/helper.rb
+- test/plugin/test_parser_openlineage.rb