fluent-plugin-openlineage 0.1.2 → 0.1.3

data/spec/tests/NominalTimeRunFacet/1.json

@@ -0,0 +1,8 @@
+{
+  "nominalTime": {
+    "_producer": "https://github.com/OpenLineage/OpenLineage/tree/0.0.1/client/python",
+    "_schemaURL": "https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/NominalTimeRunFacet",
+    "nominalEndTime": "2020-01-02",
+    "nominalStartTime": "2020-01-01"
+  }
+}

data/spec/tests/OutputStatisticsOutputDatasetFacet/1.json

@@ -0,0 +1,9 @@
+{
+  "outputStatistics": {
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/1-0-1/OpenLineage.json#/definitions/OutputStatisticsOutputDatasetFacet",
+    "rowCount": 2000,
+    "size": 2097152,
+    "fileCount": 5
+  }
+}

data/spec/tests/OwnershipDatasetFacet/1.json

@@ -0,0 +1,11 @@
+{
+  "ownership": {
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/OwnershipDatasetFacet.json",
+    "owners": [
+      {
+        "name": "owner"
+      }
+    ]
+  }
+}

data/spec/tests/OwnershipJobFacet/1.json

@@ -0,0 +1,11 @@
+{
+  "ownership": {
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/OwnershipJobFacet.json",
+    "owners": [
+      {
+        "name": "owner"
+      }
+    ]
+  }
+}

data/spec/tests/ParentRunFacet/1.json

@@ -0,0 +1,13 @@
+{
+  "parent": {
+    "_producer": "https://github.com/OpenLineage/OpenLineage/tree/0.0.1/integration/dbt",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/ParentRunFacet.json",
+    "run": {
+      "runId": "f99310b4-3c3c-1a1a-2b2b-c1b95c24ff11"
+    },
+    "job": {
+      "namespace": "dbt",
+      "name": "dbt-job-name"
+    }
+  }
+}

data/spec/tests/ProcessingEngineRunFacet/1.json

@@ -0,0 +1,9 @@
+{
+  "processing_engine": {
+    "version": "2.5.0",
+    "name": "Airflow",
+    "openlineageAdapterVersion": "0.29.2",
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/ProcessingEngineRunFacet.json"
+  }
+}

data/spec/tests/SQLJobFacet/1.json

@@ -0,0 +1,7 @@
+{
+  "sql": {
+    "_producer": "https://github.com/OpenLineage/OpenLineage/tree/0.0.1/integration/dbt",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/SQLJobFacet.json",
+    "query": "\n\nwith orders as (\n\n    select * from \"postgres\".\"public\".\"stg_orders\"\n\n),\n\npayments as (\n\n    select * from \"postgres\".\"public\".\"stg_payments\"\n\n),\n\norder_payments as (\n\n    select\n        order_id,\n\n        sum(case when payment_method = 'credit_card' then amount else 0 end) as credit_card_amount,\n        sum(case when payment_method = 'coupon' then amount else 0 end) as coupon_amount,\n        sum(case when payment_method = 'bank_transfer' then amount else 0 end) as bank_transfer_amount,\n        sum(case when payment_method = 'gift_card' then amount else 0 end) as gift_card_amount,\n        sum(amount) as total_amount\n\n    from payments\n\n    group by order_id\n\n),\n\nfinal as (\n\n    select\n        orders.order_id,\n        orders.customer_id,\n        orders.order_date,\n        orders.status,\n\n        order_payments.credit_card_amount,\n\n        order_payments.coupon_amount,\n\n        order_payments.bank_transfer_amount,\n\n        order_payments.gift_card_amount,\n\n        order_payments.total_amount as amount\n\n    from orders\n\n\n    left join order_payments\n        on orders.order_id = order_payments.order_id\n\n)\n\nselect * from final"
+  }
+}

data/spec/tests/SchemaDatasetFacet/1.json

@@ -0,0 +1,92 @@
+{
+  "schema": {
+    "fields": [
+      {
+        "name": "user_id",
+        "type": "int64"
+      },
+      {
+        "name": "counterparty_id",
+        "type": "int64"
+      },
+      {
+        "name": "currency",
+        "type": "object"
+      },
+      {
+        "name": "amount",
+        "type": "int64"
+      },
+      {
+        "name": "phones",
+        "type": "array",
+        "description": "List of phone numbers",
+        "fields": [
+          {
+            "name": "_element",
+            "type": "string",
+            "description": "Phone number"
+          }
+        ]
+      },
+      {
+        "name": "addresses",
+        "type": "struct",
+        "description": "Has customer completed activation process",
+        "fields": [
+          {
+            "name": "type",
+            "type": "string",
+            "description": "Address type, g.e. home, work, etc."
+          },
+          {
+            "name": "country",
+            "type": "string",
+            "description": "Country name"
+          },
+          {
+            "name": "zip",
+            "type": "string",
+            "description": "Zip code"
+          },
+          {
+            "name": "state",
+            "type": "string",
+            "description": "State name"
+          },
+          {
+            "name": "street",
+            "type": "string",
+            "description": "Street name"
+          }
+        ]
+      },
+      {
+        "name": "custom_properties",
+        "type": "map",
+        "fields": [
+          {
+            "name": "key",
+            "type": "string"
+          },
+          {
+            "name": "value",
+            "type": "union",
+            "fields": [
+              {
+                "name": "_0",
+                "type": "string"
+              },
+              {
+                "name": "_1",
+                "type": "int64"
+              }
+            ]
+          }
+        ]
+      }
+    ],
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-1-0/SchemaDatasetFacet.json"
+  }
+}

data/spec/tests/SourceCodeJobFacet/1.json

@@ -0,0 +1,8 @@
+{
+  "sourceCode": {
+    "language": "bash",
+    "sourceCode": "ls -halt && exit 0",
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/SourceCodeJobFacet.json"
+  }
+}

data/spec/tests/SourceCodeLocationJobFacet/1.json

@@ -0,0 +1,8 @@
+{
+  "sourceCodeLocation": {
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/SourceCodeLocationJobFacet.json",
+    "type": "git",
+    "url": "http://example.com"
+  }
+}

data/spec/tests/StorageDatasetFacet/1.json

@@ -0,0 +1,8 @@
+{
+  "storage": {
+    "storageLayer": "delta",
+    "fileFormat": "parquet",
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/StorageDatasetFacet.json"
+  }
+}

data/spec/tests/SymlinksDatasetFacet/1.json

@@ -0,0 +1,13 @@
+{
+  "symlinks": {
+    "identifiers": [
+      {
+        "namespace": "/tmp/alter_test",
+        "name": "default.alter_table_test_new",
+        "type": "TABLE"
+      }
+    ],
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/SymlinksDatasetFacet.json"
+  }
+}

data/spec/tests/example_full_event.json

@@ -0,0 +1,24 @@
+{
+  "eventTime": "2023-07-17T10:54:22.355067Z",
+  "eventType": "COMPLETE",
+  "inputs": [],
+  "job": {
+    "facets": {
+      "sql": {
+        "query": "SELECT 1",
+        "_producer": "http://123",
+        "_schemaURL": "http://123"
+      }
+    },
+    "name": "dbt",
+    "namespace": "food_delivery"
+  },
+  "outputs": [],
+  "producer": "https://github.com/OpenLineage/OpenLineage/tree/0.30.0/integration/airflow",
+  "run": {
+    "facets": {},
+    "runId": "f69a6e9b-9bac-3c9a-9cf6-eacb70ecc9a9"
+  },
+  "dataset": { "namespace": "123", "name": "1" },
+  "schemaURL": "https://openlineage.io/spec/1-0-5/OpenLineage.json#/definitions/RunEvent"
+}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-openlineage
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Pawel Leszczynski
@@ -92,7 +92,125 @@ email:
 executables: []
 extensions: []
 extra_rdoc_files: []
-files: []
+files:
+- ".github/dependabot.yml"
+- ".github/workflows/linux.yml"
+- ".gitignore"
+- ".idea/.gitignore"
+- ".idea/fluentd.iml"
+- ".idea/misc.xml"
+- ".idea/modules.xml"
+- ".idea/modules/benchmark-memory-0.2.iml"
+- ".idea/modules/bigdecimal-3.1.iml"
+- ".idea/modules/certstore_c-0.1.iml"
+- ".idea/modules/concurrent-ruby-1.3.iml"
+- ".idea/modules/concurrent-ruby-1.31.iml"
+- ".idea/modules/connection_pool-2.4.iml"
+- ".idea/modules/cool.io-1.8.iml"
+- ".idea/modules/drb-2.2.iml"
+- ".idea/modules/drb-2.21.iml"
+- ".idea/modules/ffi-1.17.iml"
+- ".idea/modules/ffi-win32-extensions-1.0.iml"
+- ".idea/modules/fluentd-1.17.iml"
+- ".idea/modules/http_parser.rb-0.8.iml"
+- ".idea/modules/json-2.7.iml"
+- ".idea/modules/json-2.71.iml"
+- ".idea/modules/msgpack-1.7.iml"
+- ".idea/modules/mutex_m-0.2.iml"
+- ".idea/modules/new_gem.iml"
+- ".idea/modules/power_assert-2.0.iml"
+- ".idea/modules/rake-13.2.iml"
+- ".idea/modules/rake-13.21.iml"
+- ".idea/modules/rake-compiler-1.2.iml"
+- ".idea/modules/rusty_json_schema-0.15.iml"
+- ".idea/modules/serverengine-2.3.iml"
+- ".idea/modules/sigdump-0.2.iml"
+- ".idea/modules/specifications.iml"
+- ".idea/modules/specifications1.iml"
+- ".idea/modules/strptime-0.2.iml"
+- ".idea/modules/thermite-0.13.iml"
+- ".idea/modules/webrick-1.8.iml"
+- ".idea/modules/win32-event-0.6.iml"
+- ".idea/modules/win32-ipc-0.7.iml"
+- ".idea/modules/yajl-ruby-1.4.iml"
+- ".rspec"
+- ChangeLog
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-openlineage.gemspec
+- lib/fluent/plugin/parser_openlineage.rb
+- misc/fluent.conf
+- misc/test-complete.json
+- misc/test-start.json
+- spec/Naming.md
+- spec/OpenLineage.json
+- spec/Versioning.md
+- spec/events/event_full.json
+- spec/events/event_invalid_dataset_facet.json
+- spec/events/event_invalid_input_dataset_facet.json
+- spec/events/event_invalid_job_facet.json
+- spec/events/event_invalid_output_dataset_facet.json
+- spec/events/event_invalid_run_facet.json
+- spec/events/event_no_run_id.json
+- spec/events/event_simple.json
+- spec/facets/ColumnLineageDatasetFacet.json
+- spec/facets/ColumnLineageDatasetFacet.md
+- spec/facets/DataQualityAssertionsDatasetFacet.json
+- spec/facets/DataQualityMetricsInputDatasetFacet.json
+- spec/facets/DatasetVersionDatasetFacet.json
+- spec/facets/DatasourceDatasetFacet.json
+- spec/facets/DocumentationDatasetFacet.json
+- spec/facets/DocumentationJobFacet.json
+- spec/facets/ErrorMessageRunFacet.json
+- spec/facets/ExternalQueryRunFacet.json
+- spec/facets/ExternalQueryRunFacet.md
+- spec/facets/ExtractionErrorRunFacet.json
+- spec/facets/JobTypeJobFacet.json
+- spec/facets/LifecycleStateChangeDatasetFacet.json
+- spec/facets/NominalTimeRunFacet.json
+- spec/facets/OutputStatisticsOutputDatasetFacet.json
+- spec/facets/OwnershipDatasetFacet.json
+- spec/facets/OwnershipJobFacet.json
+- spec/facets/ParentRunFacet.json
+- spec/facets/ProcessingEngineRunFacet.json
+- spec/facets/SQLJobFacet.json
+- spec/facets/SchemaDatasetFacet.json
+- spec/facets/SourceCodeJobFacet.json
+- spec/facets/SourceCodeLocationJobFacet.json
+- spec/facets/StorageDatasetFacet.json
+- spec/facets/SymlinksDatasetFacet.json
+- spec/fluent/plugin/test_parser_openlineage.rb
+- spec/registry/core/registry.json
+- spec/registry/gcp/facets/GcpCommonJobFacet.json
+- spec/registry/gcp/registry.json
+- spec/spec_helper.rb
+- spec/tests/ColumnLineageDatasetFacet/1.json
+- spec/tests/DataQualityAssertionsDatasetFacet/1.json
+- spec/tests/DataQualityMetricsInputDatasetFacet/1.json
+- spec/tests/DatasetVersionDatasetFacet/1.json
+- spec/tests/DatasourceDatasetFacet/1.json
+- spec/tests/DocumentationDatasetFacet/1.json
+- spec/tests/DocumentationJobFacet/1.json
+- spec/tests/ErrorMessageRunFacet/1.json
+- spec/tests/ExternalQueryRunFacet/1.json
+- spec/tests/ExtractionErrorRunFacet/1.json
+- spec/tests/JobTypeJobFacet/1.json
+- spec/tests/LifecycleStateChangeDatasetFacet/1.json
+- spec/tests/NominalTimeRunFacet/1.json
+- spec/tests/OutputStatisticsOutputDatasetFacet/1.json
+- spec/tests/OwnershipDatasetFacet/1.json
+- spec/tests/OwnershipJobFacet/1.json
+- spec/tests/ParentRunFacet/1.json
+- spec/tests/ProcessingEngineRunFacet/1.json
+- spec/tests/SQLJobFacet/1.json
+- spec/tests/SchemaDatasetFacet/1.json
+- spec/tests/SourceCodeJobFacet/1.json
+- spec/tests/SourceCodeLocationJobFacet/1.json
+- spec/tests/StorageDatasetFacet/1.json
+- spec/tests/SymlinksDatasetFacet/1.json
+- spec/tests/example_full_event.json
 homepage: http://openlineage.io
 licenses:
 - Apache-2.0
@@ -116,4 +234,71 @@ rubygems_version: 3.5.16
 signing_key:
 specification_version: 4
 summary: Parser to validate Openlineage events.
-test_files: []
+test_files:
+- spec/Naming.md
+- spec/OpenLineage.json
+- spec/Versioning.md
+- spec/events/event_full.json
+- spec/events/event_invalid_dataset_facet.json
+- spec/events/event_invalid_input_dataset_facet.json
+- spec/events/event_invalid_job_facet.json
+- spec/events/event_invalid_output_dataset_facet.json
+- spec/events/event_invalid_run_facet.json
+- spec/events/event_no_run_id.json
+- spec/events/event_simple.json
+- spec/facets/ColumnLineageDatasetFacet.json
+- spec/facets/ColumnLineageDatasetFacet.md
+- spec/facets/DataQualityAssertionsDatasetFacet.json
+- spec/facets/DataQualityMetricsInputDatasetFacet.json
+- spec/facets/DatasetVersionDatasetFacet.json
+- spec/facets/DatasourceDatasetFacet.json
+- spec/facets/DocumentationDatasetFacet.json
+- spec/facets/DocumentationJobFacet.json
+- spec/facets/ErrorMessageRunFacet.json
+- spec/facets/ExternalQueryRunFacet.json
+- spec/facets/ExternalQueryRunFacet.md
+- spec/facets/ExtractionErrorRunFacet.json
+- spec/facets/JobTypeJobFacet.json
+- spec/facets/LifecycleStateChangeDatasetFacet.json
+- spec/facets/NominalTimeRunFacet.json
+- spec/facets/OutputStatisticsOutputDatasetFacet.json
+- spec/facets/OwnershipDatasetFacet.json
+- spec/facets/OwnershipJobFacet.json
+- spec/facets/ParentRunFacet.json
+- spec/facets/ProcessingEngineRunFacet.json
+- spec/facets/SQLJobFacet.json
+- spec/facets/SchemaDatasetFacet.json
+- spec/facets/SourceCodeJobFacet.json
+- spec/facets/SourceCodeLocationJobFacet.json
+- spec/facets/StorageDatasetFacet.json
+- spec/facets/SymlinksDatasetFacet.json
+- spec/fluent/plugin/test_parser_openlineage.rb
+- spec/registry/core/registry.json
+- spec/registry/gcp/facets/GcpCommonJobFacet.json
+- spec/registry/gcp/registry.json
+- spec/spec_helper.rb
+- spec/tests/ColumnLineageDatasetFacet/1.json
+- spec/tests/DataQualityAssertionsDatasetFacet/1.json
+- spec/tests/DataQualityMetricsInputDatasetFacet/1.json
+- spec/tests/DatasetVersionDatasetFacet/1.json
+- spec/tests/DatasourceDatasetFacet/1.json
+- spec/tests/DocumentationDatasetFacet/1.json
+- spec/tests/DocumentationJobFacet/1.json
+- spec/tests/ErrorMessageRunFacet/1.json
+- spec/tests/ExternalQueryRunFacet/1.json
+- spec/tests/ExtractionErrorRunFacet/1.json
+- spec/tests/JobTypeJobFacet/1.json
+- spec/tests/LifecycleStateChangeDatasetFacet/1.json
+- spec/tests/NominalTimeRunFacet/1.json
+- spec/tests/OutputStatisticsOutputDatasetFacet/1.json
+- spec/tests/OwnershipDatasetFacet/1.json
+- spec/tests/OwnershipJobFacet/1.json
+- spec/tests/ParentRunFacet/1.json
+- spec/tests/ProcessingEngineRunFacet/1.json
+- spec/tests/SQLJobFacet/1.json
+- spec/tests/SchemaDatasetFacet/1.json
+- spec/tests/SourceCodeJobFacet/1.json
+- spec/tests/SourceCodeLocationJobFacet/1.json
+- spec/tests/StorageDatasetFacet/1.json
+- spec/tests/SymlinksDatasetFacet/1.json
+- spec/tests/example_full_event.json