fluentd-openlineage-parser 0.1.0

data/spec/registry/gcp/facets/GcpCommonJobFacet.json

@@ -0,0 +1,43 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openlineage.io/spec/facets/1-0-0/GcpCommonJobFacet.json",
+  "$defs": {
+    "GcpCommonJobFacet": {
+      "allOf": [
+        {
+          "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/JobFacet"
+        },
+        {
+          "type": "object",
+          "properties": {
+            "displayName": {
+              "description": "The name of the job to be used on UI",
+              "type": "string"
+            },
+            "origin": {
+              "type": "object",
+              "properties": {
+                "sourceType": {
+                  "description": "Type of the source. Possible values can be found in GCP documentation (https://cloud.google.com/data-catalog/docs/reference/data-lineage/rest/v1/projects.locations.processes#SourceType) ",
+                  "type": "string"
+                },
+                "name": {
+                  "description": "If the sourceType isn't CUSTOM, the value of this field should be a GCP resource name of the system, which reports lineage. The project and location parts of the resource name must match the project and location of the lineage resource being created. More details in GCP documentation https://cloud.google.com/data-catalog/docs/reference/data-lineage/rest/v1/projects.locations.processes#origin",
+                  "type": "string"
+                }
+              }
+            }
+          },
+          "additionalProperties": true
+        }
+      ],
+      "type": "object"
+    }
+  },
+  "type": "object",
+  "properties": {
+    "gcp_common": {
+      "$ref": "#/$defs/GcpCommonJobFacet"
+    }
+  }
+}

data/spec/registry/gcp/registry.json

@@ -0,0 +1,6 @@
+{
+  "consumer": {
+    "root_doc_URL": "https://cloud.google.com/data-catalog/docs/reference/data-lineage/rpc/google.cloud.datacatalog.lineage.v1#google.cloud.datacatalog.lineage.v1.Lineage.ProcessOpenLineageRunEvent",
+    "produced_facets": ["ol:gcp:GcpCommonJobFacet.json"]
+  }
+}

data/spec/release.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+#
+# Copyright 2018-2024 contributors to the OpenLineage project
+# SPDX-License-Identifier: Apache-2.0
+
+set -e
+
+function git-website() {
+  command git --git-dir "$WEBSITE_DIR/.git" --work-tree "$WEBSITE_DIR" $@
+}
+
+git config --global user.email "openlineage-bot-key@openlineage.io"
+git config --global user.name "OpenLineage deploy bot"
+
+WEBSITE_DIR=${WEBSITE_DIR:-$HOME/build/website}
+REPO="git@github.com:OpenLineage/docs"
+
+if [[ -d $WEBSITE_DIR ]]; then
+  # Check if we're in git repository and the repository points at website
+  if [[ $(git-website rev-parse --is-inside-work-tree) == "true" && $(git-website config --get remote.origin.url) == "$REPO" ]]; then
+    # Make sure we're at the head of the main branch
+    git checkout main
+    git reset --hard origin/master
+  else
+    echo "$WEBSITE_DIR is not empty - failing"
+    exit 1
+  fi
+else
+  git clone --depth 1 $REPO $WEBSITE_DIR
+fi
+
+WEBSITE_COMMIT_FILE="$WEBSITE_DIR/.last_spec_commit_id"
+
+# Check on which commit we deployed spec last
+if [[ -f $WEBSITE_COMMIT_FILE ]]; then
+  PREV_SPEC_COMMIT=$(cat "$WEBSITE_COMMIT_FILE")
+else
+  # Before lifecycle state facet
+  PREV_SPEC_COMMIT="d66c41872f3cc7f7cd5c99664d401e070e60ff48"
+fi
+
+# check if there are any changes in spec in the latest commit
+if git diff --name-only --exit-code $PREV_SPEC_COMMIT HEAD 'spec/*.json' 'spec/OpenLineage.yml' >> /dev/null; then
+  echo "no changes in spec detected, skipping publishing spec"
+  exit 0
+fi
+
+echo "Copying spec files from commit $PREV_SPEC_COMMIT"
+
+# Mark last commit on which we finished copying spec
+echo "$CIRCLE_SHA1" > "$WEBSITE_COMMIT_FILE"
+
+# Copy changed spec YML file to target location
+cp spec/OpenLineage.yml ${WEBSITE_DIR}/static/spec/OpenLineage.yml
+
+# Copy changed spec JSON files to target location
+git diff --name-only $PREV_SPEC_COMMIT HEAD 'spec/*.json' | while read LINE; do
+
+  #ignore registry files
+  if [[ $LINE =~ "registry.json" ]]; then
+      continue
+  fi
+
+  # extract target file name from $id field in spec files
+  URL=$(cat $LINE | jq -r '.["$id"]')
+
+  # extract target location in website repo
+  LOC="${WEBSITE_DIR}/static/${URL#*//*/}"
+  LOC_DIR="${LOC%/*}"
+
+  # create dir if necessary, and copy files
+  mkdir -p $LOC_DIR
+  cp $LINE $LOC
+done
+
+# commit new spec and push
+git-website add -A .last_spec_commit_id
+git-website add -A static/spec/
+git --git-dir "$WEBSITE_DIR/.git" --work-tree "$WEBSITE_DIR" commit -m "openlineage specification update"
+git-website push

data/spec/tests/ColumnLineageDatasetFacet/1.json

@@ -0,0 +1,172 @@
+{
+  "columnLineage": {
+    "fields": {
+      "NAME": {
+        "inputFields": [
+          {
+            "namespace": "SnowflakeOpenLineage",
+            "name": "CUSTOMERS",
+            "field": "NAME",
+            "transformations": [
+              {
+                "type": "DIRECT",
+                "subtype": "IDENTITY",
+                "masking": false
+              }
+            ]
+          },
+          {
+            "namespace": "SnowflakeOpenLineage",
+            "name": "CUSTOMERS",
+            "field": "ID",
+            "transformations": [
+              {
+                "type": "INDIRECT",
+                "subtype": "JOIN",
+                "description": "ON (DISCOUNTS.CUSTOMERS_ID=CUSTOMERS.ID)"
+              }
+            ]
+          },
+          {
+            "namespace": "SnowflakeOpenLineage",
+            "name": "DISCOUNTS",
+            "field": "CUSTOMERS_ID",
+            "transformations": [
+              {
+                "type": "INDIRECT",
+                "subtype": "JOIN",
+                "description": "ON (DISCOUNTS.CUSTOMERS_ID=CUSTOMERS.ID)"
+              }
+            ]
+          }
+        ],
+        "transformationDescription": "SELECT NAME",
+        "transformationType": "IDENTITY"
+      },
+      "AMOUNT_OFF": {
+        "inputFields": [
+          {
+            "namespace": "SnowflakeOpenLineage",
+            "name": "DISCOUNTS",
+            "field": "AMOUNT_OFF",
+            "transformations": [
+              {
+                "type": "DIRECT",
+                "subtype": "IDENTITY",
+                "masking": false
+              }
+            ]
+          },
+          {
+            "namespace": "SnowflakeOpenLineage",
+            "name": "CUSTOMERS",
+            "field": "ID",
+            "transformations": [
+              {
+                "type": "INDIRECT",
+                "subtype": "JOIN",
+                "description": "ON (DISCOUNTS.CUSTOMERS_ID=CUSTOMERS.ID)"
+              }
+            ]
+          },
+          {
+            "namespace": "SnowflakeOpenLineage",
+            "name": "DISCOUNTS",
+            "field": "CUSTOMERS_ID",
+            "transformations": [
+              {
+                "type": "INDIRECT",
+                "subtype": "JOIN",
+                "description": "ON (DISCOUNTS.CUSTOMERS_ID=CUSTOMERS.ID)"
+              }
+            ]
+          }
+        ],
+        "transformationType": "IDENTITY"
+      },
+      "STARTS_AT": {
+        "inputFields": [
+          {
+            "namespace": "SnowflakeOpenLineage",
+            "name": "DISCOUNTS",
+            "field": "STARTS_AT",
+            "transformations": [
+              {
+                "type": "DIRECT",
+                "subtype": "IDENTITY",
+                "masking": false
+              }
+            ]
+          },
+          {
+            "namespace": "SnowflakeOpenLineage",
+            "name": "CUSTOMERS",
+            "field": "ID",
+            "transformations": [
+              {
+                "type": "INDIRECT",
+                "subtype": "JOIN",
+                "description": "ON (DISCOUNTS.CUSTOMERS_ID=CUSTOMERS.ID)"
+              }
+            ]
+          },
+          {
+            "namespace": "SnowflakeOpenLineage",
+            "name": "DISCOUNTS",
+            "field": "CUSTOMERS_ID",
+            "transformations": [
+              {
+                "type": "INDIRECT",
+                "subtype": "JOIN",
+                "description": "ON (DISCOUNTS.CUSTOMERS_ID=CUSTOMERS.ID)"
+              }
+            ]
+          }
+        ],
+        "transformationType": "IDENTITY"
+      },
+      "ENDS_AT": {
+        "inputFields": [
+          {
+            "namespace": "SnowflakeOpenLineage",
+            "name": "DISCOUNTS",
+            "field": "ENDS_AT",
+            "transformations": [
+              {
+                "type": "DIRECT",
+                "subtype": "IDENTITY",
+                "masking": false
+              }
+            ]
+          },
+          {
+            "namespace": "SnowflakeOpenLineage",
+            "name": "CUSTOMERS",
+            "field": "ID",
+            "transformations": [
+              {
+                "type": "INDIRECT",
+                "subtype": "JOIN",
+                "description": "ON (DISCOUNTS.CUSTOMERS_ID=CUSTOMERS.ID)"
+              }
+            ]
+          },
+          {
+            "namespace": "SnowflakeOpenLineage",
+            "name": "DISCOUNTS",
+            "field": "CUSTOMERS_ID",
+            "transformations": [
+              {
+                "type": "INDIRECT",
+                "subtype": "JOIN",
+                "description": "ON (DISCOUNTS.CUSTOMERS_ID=CUSTOMERS.ID)"
+              }
+            ]
+          }
+        ]
+      }
+    },
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-1-0/ColumnLineageDatasetFacet.json"
+  }
+}

data/spec/tests/DataQualityAssertionsDatasetFacet/1.json

@@ -0,0 +1,58 @@
+{
+  "dataQualityAssertions": {
+    "assertions": [
+      {
+        "assertion": "accepted_values",
+        "column": "status",
+        "success": true
+      },
+      {
+        "assertion": "not_null",
+        "column": "amount",
+        "success": true
+      },
+      {
+        "assertion": "not_null",
+        "column": "bank_transfer_amount",
+        "success": true
+      },
+      {
+        "assertion": "not_null",
+        "column": "coupon_amount",
+        "success": true
+      },
+      {
+        "assertion": "not_null",
+        "column": "credit_card_amount",
+        "success": true
+      },
+      {
+        "assertion": "not_null",
+        "column": "customer_id",
+        "success": true
+      },
+      {
+        "assertion": "not_null",
+        "column": "gift_card_amount",
+        "success": true
+      },
+      {
+        "assertion": "not_null",
+        "column": "order_id",
+        "success": true
+      },
+      {
+        "assertion": "relationships",
+        "column": "customer_id",
+        "success": true
+      },
+      {
+        "assertion": "unique",
+        "column": "order_id",
+        "success": true
+      }
+    ],
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DataQualityAssertionsDatasetFacet.json"
+  }
+}

data/spec/tests/DataQualityMetricsInputDatasetFacet/1.json

@@ -0,0 +1,23 @@
+{
+  "dataQualityMetrics": {
+    "columnMetrics": {
+      "amount": {
+        "count": 10,
+        "max": 1500,
+        "min": 100,
+        "sum": 6600
+      },
+      "counterparty_id": {
+        "nullCount": 0
+      },
+      "user_id": {
+        "distinctCount": 3,
+        "nullCount": 0
+      }
+    },
+    "rowCount": 10,
+    "fileCount": 5,
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DataQualityMetricsInputDatasetFacet.json"
+  }
+}

data/spec/tests/DatasetVersionDatasetFacet/1.json

@@ -0,0 +1,7 @@
+{
+  "version": {
+    "datasetVersion": "2",
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DatasetVersionDatasetFacet.json"
+  }
+}

data/spec/tests/DatasourceDatasetFacet/1.json

@@ -0,0 +1,7 @@
+{
+  "version": {
+    "datasetVersion": "2",
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DatasetVersionDatasetFacet.json"
+  }
+}

data/spec/tests/DocumentationDatasetFacet/1.json

@@ -0,0 +1,7 @@
+{
+  "documentation": {
+    "description": "Some dataset description",
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DocumentationDatasetFacet.json"
+  }
+}

data/spec/tests/DocumentationJobFacet/1.json

@@ -0,0 +1,7 @@
+{
+  "documentation": {
+    "description": "Determines the popular day of week orders are placed.",
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DocumentationDatasetFacet.json"
+  }
+}

data/spec/tests/ErrorMessageRunFacet/1.json

@@ -0,0 +1,9 @@
+{
+  "errorMessage": {
+    "message": "failed",
+    "programmingLanguage": "JAVA",
+    "stackTrace": "<stack_trace>",
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/ErrorMessageRunFacet.json"
+  }
+}

data/spec/tests/ExternalQueryRunFacet/1.json

@@ -0,0 +1,8 @@
+{
+  "externalQuery": {
+    "_producer": "https://github.com/OpenLineage/OpenLineage/tree/0.0.1/integration/airflow",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/ExternalQueryRunFacet.json",
+    "externalQueryId": "bigquery_1690280798993890_f4ade238a8121576708e1892f69e67fe",
+    "source": "bigquery"
+  }
+}

data/spec/tests/ExtractionErrorRunFacet/1.json

@@ -0,0 +1,15 @@
+{
+  "extractionError": {
+    "totalTasks": 1,
+    "failedTasks": 1,
+    "errors": [
+      {
+        "errorMessage": "Expected TABLE, VIEW, INDEX, ROLE, SCHEMA, FUNCTION, STAGE or SEQUENCE after DROP, found: POLICY at Line: 1, Column 6",
+        "task": "DROP POLICY IF EXISTS name ON table_name",
+        "taskNumber": 0
+      }
+    ],
+    "_producer": "https://github.com/OpenLineage/OpenLineage/tree/0.0.1/integration/dbt",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-1/ExtractionErrorRunFacet.json"
+  }
+}

data/spec/tests/JobTypeJobFacet/1.json

@@ -0,0 +1,9 @@
+{
+  "jobType": {
+    "processingType": "BATCH",
+    "integration": "SPARK",
+    "jobType": "QUERY",
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/2-0-2/JobTypeJobFacet.json"
+  }
+}

data/spec/tests/LifecycleStateChangeDatasetFacet/1.json

@@ -0,0 +1,11 @@
+{
+  "lifecycleStateChange": {
+    "lifecycleStateChange": "RENAME",
+    "previousIdentifier": {
+      "name": "/tmp/alter_test/alter_table_test",
+      "namespace": "file"
+    },
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DatasetVersionDatasetFacet.json"
+  }
+}

data/spec/tests/NominalTimeRunFacet/1.json

@@ -0,0 +1,8 @@
+{
+  "nominalTime": {
+    "_producer": "https://github.com/OpenLineage/OpenLineage/tree/0.0.1/client/python",
+    "_schemaURL": "https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/NominalTimeRunFacet",
+    "nominalEndTime": "2020-01-02",
+    "nominalStartTime": "2020-01-01"
+  }
+}

data/spec/tests/OutputStatisticsOutputDatasetFacet/1.json

@@ -0,0 +1,9 @@
+{
+  "outputStatistics": {
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/1-0-1/OpenLineage.json#/definitions/OutputStatisticsOutputDatasetFacet",
+    "rowCount": 2000,
+    "size": 2097152,
+    "fileCount": 5
+  }
+}

data/spec/tests/OwnershipDatasetFacet/1.json

@@ -0,0 +1,11 @@
+{
+  "ownership": {
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/OwnershipDatasetFacet.json",
+    "owners": [
+      {
+        "name": "owner"
+      }
+    ]
+  }
+}

data/spec/tests/OwnershipJobFacet/1.json

@@ -0,0 +1,11 @@
+{
+  "ownership": {
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/OwnershipJobFacet.json",
+    "owners": [
+      {
+        "name": "owner"
+      }
+    ]
+  }
+}

data/spec/tests/ParentRunFacet/1.json

@@ -0,0 +1,13 @@
+{
+  "parent": {
+    "_producer": "https://github.com/OpenLineage/OpenLineage/tree/0.0.1/integration/dbt",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/ParentRunFacet.json",
+    "run": {
+      "runId": "f99310b4-3c3c-1a1a-2b2b-c1b95c24ff11"
+    },
+    "job": {
+      "namespace": "dbt",
+      "name": "dbt-job-name"
+    }
+  }
+}

data/spec/tests/ProcessingEngineRunFacet/1.json

@@ -0,0 +1,9 @@
+{
+  "processing_engine": {
+    "version": "2.5.0",
+    "name": "Airflow",
+    "openlineageAdapterVersion": "0.29.2",
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/ProcessingEngineRunFacet.json"
+  }
+}

data/spec/tests/SQLJobFacet/1.json

@@ -0,0 +1,7 @@
+{
+  "sql": {
+    "_producer": "https://github.com/OpenLineage/OpenLineage/tree/0.0.1/integration/dbt",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/SQLJobFacet.json",
+    "query": "\n\nwith orders as (\n\n    select * from \"postgres\".\"public\".\"stg_orders\"\n\n),\n\npayments as (\n\n    select * from \"postgres\".\"public\".\"stg_payments\"\n\n),\n\norder_payments as (\n\n    select\n        order_id,\n\n        sum(case when payment_method = 'credit_card' then amount else 0 end) as credit_card_amount,\n        sum(case when payment_method = 'coupon' then amount else 0 end) as coupon_amount,\n        sum(case when payment_method = 'bank_transfer' then amount else 0 end) as bank_transfer_amount,\n        sum(case when payment_method = 'gift_card' then amount else 0 end) as gift_card_amount,\n        sum(amount) as total_amount\n\n    from payments\n\n    group by order_id\n\n),\n\nfinal as (\n\n    select\n        orders.order_id,\n        orders.customer_id,\n        orders.order_date,\n        orders.status,\n\n        order_payments.credit_card_amount,\n\n        order_payments.coupon_amount,\n\n        order_payments.bank_transfer_amount,\n\n        order_payments.gift_card_amount,\n\n        order_payments.total_amount as amount\n\n    from orders\n\n\n    left join order_payments\n        on orders.order_id = order_payments.order_id\n\n)\n\nselect * from final"
+  }
+}

data/spec/tests/SchemaDatasetFacet/1.json

@@ -0,0 +1,92 @@
+{
+  "schema": {
+    "fields": [
+      {
+        "name": "user_id",
+        "type": "int64"
+      },
+      {
+        "name": "counterparty_id",
+        "type": "int64"
+      },
+      {
+        "name": "currency",
+        "type": "object"
+      },
+      {
+        "name": "amount",
+        "type": "int64"
+      },
+      {
+        "name": "phones",
+        "type": "array",
+        "description": "List of phone numbers",
+        "fields": [
+          {
+            "name": "_element",
+            "type": "string",
+            "description": "Phone number"
+          }
+        ]
+      },
+      {
+        "name": "addresses",
+        "type": "struct",
+        "description": "Has customer completed activation process",
+        "fields": [
+          {
+            "name": "type",
+            "type": "string",
+            "description": "Address type, g.e. home, work, etc."
+          },
+          {
+            "name": "country",
+            "type": "string",
+            "description": "Country name"
+          },
+          {
+            "name": "zip",
+            "type": "string",
+            "description": "Zip code"
+          },
+          {
+            "name": "state",
+            "type": "string",
+            "description": "State name"
+          },
+          {
+            "name": "street",
+            "type": "string",
+            "description": "Street name"
+          }
+        ]
+      },
+      {
+        "name": "custom_properties",
+        "type": "map",
+        "fields": [
+          {
+            "name": "key",
+            "type": "string"
+          },
+          {
+            "name": "value",
+            "type": "union",
+            "fields": [
+              {
+                "name": "_0",
+                "type": "string"
+              },
+              {
+                "name": "_1",
+                "type": "int64"
+              }
+            ]
+          }
+        ]
+      }
+    ],
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-1-0/SchemaDatasetFacet.json"
+  }
+}

data/spec/tests/SourceCodeJobFacet/1.json

@@ -0,0 +1,8 @@
+{
+  "sourceCode": {
+    "language": "bash",
+    "sourceCode": "ls -halt && exit 0",
+    "_producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client",
+    "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/SourceCodeJobFacet.json"
+  }
+}