fluent-plugin-openlineage 0.1.2 → 0.1.3

data/lib/fluent/plugin/parser_openlineage.rb

@@ -0,0 +1,182 @@
+require "fluent/plugin/parser"
+require 'fluent/plugin/parser_json'
+require 'json'
+require "rusty_json_schema"
+
+module Fluent
+  module Plugin
+    class OpenlineageParser < Fluent::Plugin::JSONParser
+      Fluent::Plugin.register_parser("openlineage", self)
+
+      DEFAULT_SPEC_DIRECTORY="/etc/spec"
+
+      def configure(conf)
+        if conf.has_key?('spec_directory')
+          @spec_directory = conf['spec_directory']
+        else
+          @spec_directory = DEFAULT_SPEC_DIRECTORY
+        end
+        if (not @spec_directory.end_with?("/"))
+          @spec_directory += "/"
+        end
+        @validate_input_dataset_facets = conf.fetch('validate_input_dataset_facets', false)
+        @validate_output_dataset_facets = conf.fetch('validate_output_dataset_facets', false)
+        @validate_dataset_facets = conf.fetch('validate_dataset_facets', false)
+        @validate_run_facets = conf.fetch('validate_run_facets', true)
+        @validate_job_facets = conf.fetch('validate_job_facets', true)
+        @schema = load_schema()
+        @validator = RustyJSONSchema.build(@schema)
+        super
+      end
+
+      # https://docs.fluentd.org/plugin-development/api-plugin-parser
+      def parse(text)
+        # parse JSON with default JSONParser
+        super(text) { | time, json |
+          validate_openlineage(json)
+          yield time, json
+        }
+      end
+
+      private
+
+
+      def validate_openlineage(json)
+        if json == nil
+          raise ParserError, "Openlineage validation failed: invalid json provided"
+        end
+
+        # https://github.com/driv3r/rusty_json_schema
+        # Rust json parser ported to ruby that supports Draft 2020-12
+        errors = @validator.validate(json)
+
+        if errors.join(", ").include? "is not valid under any of the given schemas"
+          errors = enrich_oneOf_errors(json)
+        end
+        if !errors.empty?
+          raise ParserError, "Openlineage validation failed: " + errors.join(", ")
+        end
+      end
+
+      # Validator returns very generic OneOfNotValid error message
+      # We try to find better reason for mismatch with each candidate.
+      def enrich_oneOf_errors(json)
+        errors = []
+        @schema["oneOf"].each { |ref|
+          changed_schema = Marshal.load(Marshal.dump(@schema))
+          changed_schema.delete("oneOf")
+          changed_schema["$ref"] = ref["$ref"]
+          validator = RustyJSONSchema.build(changed_schema)
+          error = validator.validate(json)
+          if !error.empty?
+            errors.append("#{ref}: #{error.join(", ")}")
+          end
+        }
+        return errors
+      end
+
+      def load_schema()
+        schemaFile = @spec_directory + "OpenLineage.json"
+
+        if (not File.exist?(schemaFile))
+          raise ParserError, "Couldn't find Openlineage.json file within a defined spec directory: " + schemaFile
+        end
+
+        schema = File.read(schemaFile)
+        schema = rewrite_schema_to_include_facets(schema)
+        return schema
+      end
+
+      # Current Openlineage schema contains references to facets' definitions stored in files
+      # in facets directory which are not valid schemas for json_schema.
+      # In this step we rewrite Openlineage schema to contain facets definitions within it
+      def rewrite_schema_to_include_facets(schema)
+        # replace all the refs in schema to local refs
+        # "facets/ColumnLineageDatasetFacet.json" -> "#/defs/ColumnLineageDatasetFacet"
+        schema = schema.gsub(
+          /"facets\/([a-zA-Z]+)\.json"/,
+          '"#/$defs/\1"'
+        )
+        schema_json = JSON.parse(schema)
+        facets_path = @spec_directory + "facets/"
+
+        
+        # list all the facets
+        Dir.glob("#{facets_path}/*.json").each { |facet_file|
+            facet_schema =  JSON.parse(
+              File.read(facet_file).gsub(
+                /"https:\/\/openlineage\.io\/spec\/\d-\d-\d\/OpenLineage\.json#\/\$defs\/([a-zA-Z]+)"/,
+                '"#/$defs/\1"'
+              )
+            )
+
+            facet_schema["properties"].each { |property, ref|
+              facet_name =  ref["$ref"]&.gsub("#/$defs/", "")
+              parents = []
+              facet_schema["$defs"][facet_name]["allOf"]&.each { |definition|
+                unless definition["$ref"].nil?
+                  parents.append(definition["$ref"].gsub("#/$defs/", ""))
+                end
+              }
+              parents.each {|parent|
+              add_ref_as_parent_property(schema_json, parent, facet_name, property)
+              }
+            }
+            # include facets' definitions within schema
+            schema_json["$defs"] = schema_json["$defs"].merge(facet_schema["$defs"])
+        }
+        return schema_json
+      end
+
+      def add_ref_as_parent_property(schema, parent, facet_name, property)
+        getter = find_parent_object_getter(parent)
+        if getter.nil?
+          return
+        end
+        properties = getter.call(schema)["properties"] || {}
+        properties[property] = {"$ref" => "#/$defs/" + facet_name}
+        getter.call(schema)["properties"] = properties
+      end
+
+      # Based on facet name find path to object facets
+      def find_parent_object_getter(parent)
+        getter = nil
+        case parent
+        when "JobFacet"
+          if @validate_job_facets
+            getter = ->(schema) { schema["$defs"]["Job"]["properties"]["facets"] }
+          end
+        when "RunFacet"
+          if @validate_run_facets
+            getter = ->(schema) { schema["$defs"]["Run"]["properties"]["facets"] }
+          end
+        when "DatasetFacet"
+          if @validate_dataset_facets
+            getter = ->(schema) { schema["$defs"]["Dataset"]["properties"]["facets"] }
+          end
+        when "OutputDatasetFacet"
+          if @validate_output_dataset_facets
+            getter = ->(schema) { schema \
+            ["$defs"] \
+            ["OutputDataset"] \
+            ["allOf"].select {|el| el.key?("type") } \
+            [0] \
+            ["properties"] \
+            ["outputFacets"] }
+          end
+        when "InputDatasetFacet"
+          if @validate_input_dataset_facets
+            getter = ->(schema) { schema \
+            ["$defs"] \
+            ["InputDataset"] \
+            ["allOf"].select {|el| el.key?("type") } \
+            [0] \
+            ["properties"] \
+            ["inputFacets"] }
+          end
+        end
+        return getter
+      end
+    end
+  end
+end

data/misc/fluent.conf

@@ -0,0 +1,101 @@
+<source>
+  @type http
+  port 9880
+  <parse>
+    @type openlineage
+  </parse>
+</source>
+
+
+# https://docs.fluentd.org/output/http
+<match api.v1.lineage> # tag should match fluentd input endpoint url http://localhost:9880/api/v1/lineage
+  @type copy
+  <store>
+    @type http
+    endpoint_url "#{ENV['MARQUEZ_HTTP_ENDPOINT']}"
+    content_type application/json
+    bulk_request false # available since using https://github.com/fluent-plugins-nursery/fluent-plugin-out-http
+    buffered true
+    serializer json
+    retryable_response_codes 408, 429, 500, 502, 503
+
+    <buffer>
+      @type file
+      path /tmp/openlineage/buf/chunk-*
+      flush_mode immediate
+    </buffer>
+  </store>
+
+  <store>
+    @type stdout # testing purpose to demonstrate that copy is working
+  </store>
+
+  # other output stores can be put
+</match>
+
+
+# source for prometheus metrics
+<source>
+  @type forward
+  bind 0.0.0.0
+  port 24224
+</source>
+
+# count the number of incoming records per tag
+<filter company.*>
+  @type prometheus
+  <metric>
+    name fluentd_input_status_num_records_total
+    type counter
+    desc The total number of incoming records
+    <labels>
+      tag ${tag}
+      hostname ${hostname}
+    </labels>
+  </metric>
+</filter>
+
+# count the number of outgoing records per tag
+<match company.*>
+  @type copy
+
+  <store>
+    @type forward
+    <server>
+      name myserver1
+      host 192.168.1.3
+      port 24224
+      weight 60
+    </server>
+  </store>
+
+  <store>
+    @type prometheus
+    <metric>
+      name fluentd_output_status_num_records_total
+      type counter
+      desc The total number of outgoing records
+      <labels>
+        tag ${tag}
+        hostname ${hostname}
+      </labels>
+    </metric>
+  </store>
+
+</match>
+
+# expose metrics in prometheus format
+<source>
+  @type prometheus
+  bind 0.0.0.0
+  port 24231
+  metrics_path /metrics
+</source>
+
+<source>
+  @type prometheus_output_monitor
+  interval 10
+  <labels>
+    hostname ${hostname}
+  </labels>
+</source>

data/misc/test-complete.json

@@ -0,0 +1,73 @@
+{
+  "eventType": "COMPLETE",
+  "eventTime": "2019-05-09T19:50:24.201361Z",
+  "run": {
+    "runId": "d46e465b-d358-4d32-83d4-df660ff614dd"
+  },
+  "job": {
+    "namespace": "my-namespace",
+    "name": "my-job"
+  },
+  "inputs": [
+    {
+      "namespace": "my-namespace",
+      "name": "my-input",
+      "facets": {
+        "schema": {
+          "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+          "_schemaURL": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/spec/OpenLineage.json#/definitions/SchemaDatasetFacet",
+          "fields": [
+            {
+              "name": "a",
+              "type": "INTEGER"
+            },
+            {
+              "name": "b",
+              "type": "TIMESTAMP"
+            },
+            {
+              "name": "c",
+              "type": "INTEGER"
+            },
+            {
+              "name": "d",
+              "type": "INTEGER"
+            }
+          ]
+        }
+      }
+    }
+  ],
+  "outputs": [
+    {
+      "namespace": "my-namespace",
+      "name": "my-output",
+      "facets": {
+        "schema": {
+          "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+          "_schemaURL": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/spec/OpenLineage.json#/definitions/SchemaDatasetFacet",
+          "fields": [
+            {
+              "name": "a",
+              "type": "INTEGER"
+            },
+            {
+              "name": "b",
+              "type": "TIMESTAMP"
+            },
+            {
+              "name": "c",
+              "type": "INTEGER"
+            },
+            {
+              "name": "d",
+              "type": "INTEGER"
+            }
+          ]
+        }
+      }
+    }
+  ],
+  "producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+  "schemaURL": "https://openlineage.io/spec/1-0-1/OpenLineage.json#/definitions/RunEvent"
+}

data/misc/test-start.json

@@ -0,0 +1,73 @@
+{
+  "eventType": "START",
+  "eventTime": "2019-05-09T19:49:24.201361Z",
+  "run": {
+    "runId": "d46e465b-d358-4d32-83d4-df660ff614dd"
+  },
+  "job": {
+    "namespace": "my-namespace",
+    "name": "my-job"
+  },
+  "inputs": [
+    {
+      "namespace": "my-namespace",
+      "name": "my-input",
+      "facets": {
+        "schema": {
+          "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+          "_schemaURL": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/spec/OpenLineage.json#/definitions/SchemaDatasetFacet",
+          "fields": [
+            {
+              "name": "a",
+              "type": "INTEGER"
+            },
+            {
+              "name": "b",
+              "type": "TIMESTAMP"
+            },
+            {
+              "name": "c",
+              "type": "INTEGER"
+            },
+            {
+              "name": "d",
+              "type": "INTEGER"
+            }
+          ]
+        }
+      }
+    }
+  ],
+  "outputs": [
+    {
+      "namespace": "my-namespace",
+      "name": "my-output",
+      "facets": {
+        "schema": {
+          "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+          "_schemaURL": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/spec/OpenLineage.json#/definitions/SchemaDatasetFacet",
+          "fields": [
+            {
+              "name": "a",
+              "type": "INTEGER"
+            },
+            {
+              "name": "b",
+              "type": "TIMESTAMP"
+            },
+            {
+              "name": "c",
+              "type": "INTEGER"
+            },
+            {
+              "name": "d",
+              "type": "INTEGER"
+            }
+          ]
+        }
+      }
+    }
+  ],
+  "producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+  "schemaURL": "https://openlineage.io/spec/1-0-1/OpenLineage.json#/definitions/RunEvent"
+}