opencode-skills-collection 2.0.0 → 2.0.3

package/bundled-skills/monte-carlo-push-ingestion/scripts/sample_verify.py

@@ -0,0 +1,357 @@
+#!/usr/bin/env python3
+"""
+Monte Carlo Push Ingestion — Verification Helper
+
+Queries the Monte Carlo GraphQL API to verify that pushed metadata, lineage, and
+query logs are visible in the platform.
+
+Prerequisites:
+    pip install requests
+
+    Set environment variables:
+        MCD_ID      — GraphQL API key ID   (from getmontecarlo.com/settings/api)
+        MCD_TOKEN   — GraphQL API key secret
+        MCD_RESOURCE_UUID   — Your MC warehouse/resource UUID
+
+Usage:
+    python sample_verify.py \
+        --full-table-id "analytics:public.orders" \
+        --check-schema \
+        --check-metrics \
+        --check-detectors \
+        --check-lineage \
+        --expected-sources "analytics:public.customers" "analytics:public.raw_orders"
+"""
+
+import argparse
+import json
+import os
+import sys
+from datetime import datetime, timedelta, timezone
+
+import requests
+
+GRAPHQL_URL = "https://api.getmontecarlo.com/graphql"
+
+
+def graphql(query: str, variables: dict, key_id: str, key_token: str) -> dict:
+    """Execute a GraphQL query/mutation and return the data payload."""
+    resp = requests.post(
+        GRAPHQL_URL,
+        json={"query": query, "variables": variables},
+        headers={
+            "x-mcd-id": key_id,
+            "x-mcd-token": key_token,
+            "Content-Type": "application/json",
+        },
+        timeout=30,
+    )
+    resp.raise_for_status()
+    body = resp.json()
+    if "errors" in body:
+        raise RuntimeError(json.dumps(body["errors"], indent=2))
+    return body["data"]
+
+
+# ---------------------------------------------------------------------------
+# Step 1: Resolve MCON from fullTableId
+# ---------------------------------------------------------------------------
+
+def get_table_mcon(full_table_id: str, dw_id: str, key_id: str, key_token: str) -> str:
+    """Resolve a fullTableId + warehouse UUID to an MCON."""
+    data = graphql(
+        """query GetTable($fullTableId: String!, $dwId: UUID!) {
+             getTable(fullTableId: $fullTableId, dwId: $dwId) {
+               mcon fullTableId displayName
+             }
+           }""",
+        {"fullTableId": full_table_id, "dwId": dw_id},
+        key_id, key_token,
+    )
+    table = data.get("getTable")
+    if not table:
+        raise ValueError(f"Table not found: {full_table_id} in resource {dw_id}")
+    print(f"  Resolved: {table['fullTableId']} → MCON: {table['mcon']}")
+    return table["mcon"]
+
+
+# ---------------------------------------------------------------------------
+# Step 2: Verify schema (columns)
+# ---------------------------------------------------------------------------
+
+def verify_schema(mcon: str, expected_fields: list[str], key_id: str, key_token: str) -> bool:
+    """Check that the table's column names match expected_fields."""
+    data = graphql(
+        """query GetSchema($mcon: String!) {
+             getTable(mcon: $mcon) {
+               versions {
+                 edges {
+                   node {
+                     fields { name fieldType }
+                   }
+                 }
+               }
+             }
+           }""",
+        {"mcon": mcon},
+        key_id, key_token,
+    )
+    edges = (data.get("getTable") or {}).get("versions", {}).get("edges", [])
+    if not edges:
+        print("  WARN: no schema versions found")
+        return False
+    fields = edges[0]["node"]["fields"]
+    got_names = {f["name"].lower() for f in fields}
+    print(f"  Schema: {len(fields)} column(s) — {', '.join(f['name'] for f in fields[:8])}{'...' if len(fields) > 8 else ''}")
+    if expected_fields:
+        missing = [e for e in expected_fields if e.lower() not in got_names]
+        if missing:
+            print(f"  FAIL: missing columns: {missing}")
+            return False
+        print(f"  PASS: all expected columns present")
+    return True
+
+
+# ---------------------------------------------------------------------------
+# Step 3: Verify volume/freshness metrics
+# ---------------------------------------------------------------------------
+
+def verify_metrics(mcon: str, key_id: str, key_token: str) -> None:
+    """Fetch and display the latest row_count and freshness metrics."""
+    end = datetime.now(tz=timezone.utc)
+    start = end - timedelta(days=7)
+    for metric_name in ("total_row_count", "total_row_count_last_changed_on"):
+        data = graphql(
+            """query GetMetrics($mcon: String!, $metricName: String!, $start: DateTime!, $end: DateTime!) {
+                 getMetricsV4(dwId: null, mcon: $mcon, metricName: $metricName,
+                              startTime: $start, endTime: $end) {
+                   metricsJson
+                 }
+               }""",
+            {"mcon": mcon, "metricName": metric_name,
+             "start": start.isoformat(), "end": end.isoformat()},
+            key_id, key_token,
+        )
+        metrics_json = (data.get("getMetricsV4") or {}).get("metricsJson")
+        if not metrics_json:
+            print(f"  {metric_name}: no data")
+            continue
+        points = json.loads(metrics_json)
+        if not points:
+            print(f"  {metric_name}: no data points")
+            continue
+        latest = max(points, key=lambda p: p.get("measurementTimestamp") or "")
+        val = latest.get("value")
+        ts = latest.get("measurementTimestamp")
+        if metric_name == "total_row_count_last_changed_on" and val:
+            ts_fmt = datetime.fromtimestamp(float(val), tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+            print(f"  {metric_name}: {ts_fmt}")
+        else:
+            print(f"  {metric_name}: {val} (at {ts})")
+
+
+# ---------------------------------------------------------------------------
+# Step 3b: Verify detector status (freshness + volume)
+# ---------------------------------------------------------------------------
+
+def verify_detectors(mcon: str, key_id: str, key_token: str) -> None:
+    """Check the status of freshness and volume anomaly detectors."""
+    data = graphql(
+        """query GetDetectors($mcon: String!) {
+             getTable(mcon: $mcon) {
+               thresholds {
+                 freshness { status }
+                 size { status }
+               }
+             }
+           }""",
+        {"mcon": mcon},
+        key_id, key_token,
+    )
+    thresholds = (data.get("getTable") or {}).get("thresholds") or {}
+    freshness = thresholds.get("freshness") or {}
+    size = thresholds.get("size") or {}
+    freshness_status = freshness.get("status", "not available")
+    size_status = size.get("status", "not available")
+    print(f"  Freshness detector: {freshness_status}")
+    print(f"  Volume detector:    {size_status}")
+    if freshness_status in ("no data", "training"):
+        print("  ↳ Freshness needs 7+ pushes with changed last_update_time over ~2 weeks")
+    if size_status in ("no data", "training"):
+        print("  ↳ Volume needs 10-48 samples over ~42 days (push hourly, consistently)")
+
+
+# ---------------------------------------------------------------------------
+# Step 4: Verify table lineage (upstream)
+# ---------------------------------------------------------------------------
+
+def verify_table_lineage(
+    mcon: str,
+    expected_source_mcons: list[str],
+    key_id: str,
+    key_token: str,
+) -> bool:
+    """Check that expected source MCONs appear in the upstream lineage."""
+    data = graphql(
+        """query GetLineage($mcon: String!) {
+             getTableLineage(mcon: $mcon, direction: "upstream", hops: 1) {
+               connectedNodes { mcon displayName objectType }
+               flattenedEdges { directlyConnectedMcons }
+             }
+           }""",
+        {"mcon": mcon},
+        key_id, key_token,
+    )
+    lineage = data.get("getTableLineage") or {}
+    connected = {n["mcon"] for n in lineage.get("connectedNodes", [])}
+    flat = {m for e in lineage.get("flattenedEdges", []) for m in e.get("directlyConnectedMcons", [])}
+    all_found = connected | flat
+    print(f"  Upstream nodes: {len(connected)}")
+    if not expected_source_mcons:
+        return True
+    missing = [s for s in expected_source_mcons if s not in all_found]
+    if missing:
+        print(f"  FAIL: missing sources: {missing}")
+        return False
+    print("  PASS: all expected sources present")
+    return True
+
+
+# ---------------------------------------------------------------------------
+# Step 5: Verify column lineage
+# ---------------------------------------------------------------------------
+
+def verify_column_lineage(
+    source_mcon: str,
+    source_column: str,
+    expected_dest_mcon: str,
+    expected_dest_column: str,
+    key_id: str,
+    key_token: str,
+) -> bool:
+    """Check that source_column flows to expected_dest_column on expected_dest_mcon."""
+    data = graphql(
+        """query GetColLineage($mcon: String!, $column: String!) {
+             getDerivedTablesPartialLineage(mcon: $mcon, column: $column, pageSize: 1000) {
+               destinations {
+                 table { mcon displayName }
+                 columns { columnName }
+               }
+             }
+           }""",
+        {"mcon": source_mcon, "column": source_column},
+        key_id, key_token,
+    )
+    destinations = (data.get("getDerivedTablesPartialLineage") or {}).get("destinations", [])
+    for dest in destinations:
+        if dest["table"]["mcon"] == expected_dest_mcon:
+            cols = {c["columnName"] for c in dest.get("columns", [])}
+            if expected_dest_column in cols:
+                print(f"  PASS: {source_column} → {dest['table']['displayName']}.{expected_dest_column}")
+                return True
+    print(f"  FAIL: {source_column} → {expected_dest_mcon}.{expected_dest_column} not found")
+    return False
+
+
+# ---------------------------------------------------------------------------
+# Step 6: Verify query logs
+# ---------------------------------------------------------------------------
+
+def verify_query_logs(
+    mcon: str,
+    start_time: datetime,
+    end_time: datetime,
+    key_id: str,
+    key_token: str,
+) -> None:
+    """Report read/write query counts for a table within the given time window."""
+    for query_type in ("read", "write"):
+        cursor = None
+        total = 0
+        while True:
+            data = graphql(
+                """query GetQueries($mcon: String!, $type: String!, $start: DateTime!, $end: DateTime!, $after: String) {
+                     getAggregatedQueries(mcon: $mcon, queryType: $type,
+                                         startTime: $start, endTime: $end,
+                                         first: 200, after: $after) {
+                       edges { node { queryHash queryCount lastSeen } }
+                       pageInfo { hasNextPage endCursor }
+                     }
+                   }""",
+                {"mcon": mcon, "type": query_type,
+                 "start": start_time.isoformat(), "end": end_time.isoformat(),
+                 "after": cursor},
+                key_id, key_token,
+            )
+            result = data.get("getAggregatedQueries") or {}
+            total += sum(e["node"]["queryCount"] for e in result.get("edges", []))
+            page = result.get("pageInfo", {})
+            if not page.get("hasNextPage"):
+                break
+            cursor = page["endCursor"]
+        print(f"  {query_type} queries: {total}")
+
+
+# ---------------------------------------------------------------------------
+# CLI
+# ---------------------------------------------------------------------------
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Verify Monte Carlo push-ingested data via GraphQL")
+    parser.add_argument("--key-id",    default=os.environ.get("MCD_ID"))
+    parser.add_argument("--key-token", default=os.environ.get("MCD_TOKEN"))
+    parser.add_argument("--resource-uuid", default=os.environ.get("MCD_RESOURCE_UUID"), required=False)
+    parser.add_argument("--full-table-id", required=True, help="e.g. analytics:public.orders")
+    parser.add_argument("--mcon", help="Use MCON directly instead of resolving from fullTableId")
+    parser.add_argument("--check-schema",  action="store_true")
+    parser.add_argument("--check-metrics", action="store_true")
+    parser.add_argument("--check-detectors", action="store_true", help="Check freshness/volume detector status")
+    parser.add_argument("--check-lineage", action="store_true")
+    parser.add_argument("--check-query-logs", action="store_true")
+    parser.add_argument("--expected-fields", nargs="*", default=[])
+    parser.add_argument("--expected-sources", nargs="*", default=[], help="Source MCONs for lineage check")
+    parser.add_argument("--lookback-hours", type=int, default=24, help="For query log check (default: 24)")
+    args = parser.parse_args()
+
+    if not args.key_id or not args.key_token:
+        print("ERROR: Provide --key-id/--key-token or set MCD_ID/MCD_TOKEN", file=sys.stderr)
+        sys.exit(1)
+
+    print(f"\n{'='*60}")
+    print(f"Verifying: {args.full_table_id}")
+    print(f"{'='*60}")
+
+    mcon = args.mcon
+    if not mcon:
+        if not args.resource_uuid:
+            print("ERROR: --resource-uuid required when --mcon is not provided", file=sys.stderr)
+            sys.exit(1)
+        mcon = get_table_mcon(args.full_table_id, args.resource_uuid, args.key_id, args.key_token)
+
+    if args.check_schema:
+        print("\n[Schema]")
+        verify_schema(mcon, args.expected_fields, args.key_id, args.key_token)
+
+    if args.check_metrics:
+        print("\n[Metrics]")
+        verify_metrics(mcon, args.key_id, args.key_token)
+
+    if args.check_detectors:
+        print("\n[Detectors]")
+        verify_detectors(mcon, args.key_id, args.key_token)
+
+    if args.check_lineage:
+        print("\n[Table Lineage]")
+        verify_table_lineage(mcon, args.expected_sources, args.key_id, args.key_token)
+
+    if args.check_query_logs:
+        print("\n[Query Logs]")
+        end = datetime.now(tz=timezone.utc)
+        start = end - timedelta(hours=args.lookback_hours)
+        verify_query_logs(mcon, start, end, args.key_id, args.key_token)
+
+    print("\nDone.")
+
+
+if __name__ == "__main__":
+    main()

package/bundled-skills/monte-carlo-push-ingestion/scripts/templates/bigquery/collect_and_push_lineage.py

@@ -0,0 +1,70 @@
+"""
+BigQuery — Lineage Collection and Push (combined)
+===================================================
+Imports ``collect()`` from ``collect_lineage`` and ``push()`` from
+``push_lineage``, runs both in sequence.
+
+Substitution points (search for "← SUBSTITUTE"):
+  - BIGQUERY_PROJECT_ID   : GCP project ID to collect from
+  - BIGQUERY_REGION       : BigQuery region for INFORMATION_SCHEMA queries (e.g. "us", "eu")
+  - LOOKBACK_HOURS        : how far back to scan job history (default 24 h)
+  - MCD_INGEST_ID / MCD_INGEST_TOKEN : Monte Carlo API credentials
+  - MCD_RESOURCE_UUID      : UUID of the BigQuery connection in Monte Carlo
+
+Prerequisites:
+  pip install google-cloud-bigquery pycarlo
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+
+from collect_lineage import collect, LOOKBACK_HOURS
+from push_lineage import push, _BATCH_SIZE
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Push BigQuery lineage to Monte Carlo")
+    parser.add_argument("--project-id", default=os.getenv("BIGQUERY_PROJECT_ID"))  # ← SUBSTITUTE
+    parser.add_argument("--region", default=os.getenv("BIGQUERY_REGION", "us"))    # ← SUBSTITUTE
+    parser.add_argument("--resource-uuid", default=os.getenv("MCD_RESOURCE_UUID"))
+    parser.add_argument("--key-id", default=os.getenv("MCD_INGEST_ID"))
+    parser.add_argument("--key-token", default=os.getenv("MCD_INGEST_TOKEN"))
+    parser.add_argument("--lookback-hours", type=int, default=LOOKBACK_HOURS)
+    parser.add_argument("--output-file", default="lineage_output.json")
+    parser.add_argument("--push-result-file", default="lineage_push_result.json")
+    parser.add_argument(
+        "--batch-size",
+        type=int,
+        default=_BATCH_SIZE,
+        help=f"Max events per push batch (default: {_BATCH_SIZE})",
+    )
+    args = parser.parse_args()
+
+    required = ["project_id", "resource_uuid", "key_id", "key_token"]
+    missing = [k for k in required if getattr(args, k) is None]
+    if missing:
+        parser.error(f"Missing required arguments/env vars: {missing}")
+
+    # Step 1: Collect
+    collect(
+        project_id=args.project_id,
+        region=args.region,
+        lookback_hours=args.lookback_hours,
+        output_file=args.output_file,
+    )
+
+    # Step 2: Push
+    push(
+        input_file=args.output_file,
+        resource_uuid=args.resource_uuid,
+        key_id=args.key_id,
+        key_token=args.key_token,
+        batch_size=args.batch_size,
+        output_file=args.push_result_file,
+    )
+
+
+if __name__ == "__main__":
+    main()

package/bundled-skills/monte-carlo-push-ingestion/scripts/templates/bigquery/collect_and_push_metadata.py

@@ -0,0 +1,65 @@
+"""
+BigQuery — Metadata Collection and Push (combined)
+===================================================
+Imports ``collect()`` from ``collect_metadata`` and ``push()`` from
+``push_metadata``, runs both in sequence.
+
+Substitution points (search for "← SUBSTITUTE"):
+  - BIGQUERY_PROJECT_ID   : GCP project ID to collect from
+  - GOOGLE_APPLICATION_CREDENTIALS : path to service-account JSON key file
+  - MCD_INGEST_ID / MCD_INGEST_TOKEN : Monte Carlo API credentials
+  - MCD_RESOURCE_UUID      : UUID of the BigQuery connection in Monte Carlo
+  - DATASET_EXCLUSIONS    : datasets to skip (informational / system datasets)
+
+Prerequisites:
+  pip install google-cloud-bigquery pycarlo
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+
+from collect_metadata import collect
+from push_metadata import push, _BATCH_SIZE
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Push BigQuery metadata to Monte Carlo")
+    parser.add_argument("--project-id", default=os.getenv("BIGQUERY_PROJECT_ID"))  # ← SUBSTITUTE
+    parser.add_argument("--resource-uuid", default=os.getenv("MCD_RESOURCE_UUID"))
+    parser.add_argument("--key-id", default=os.getenv("MCD_INGEST_ID"))
+    parser.add_argument("--key-token", default=os.getenv("MCD_INGEST_TOKEN"))
+    parser.add_argument("--output-file", default="metadata_output.json")
+    parser.add_argument("--push-result-file", default="metadata_push_result.json")
+    parser.add_argument(
+        "--batch-size",
+        type=int,
+        default=_BATCH_SIZE,
+        help=f"Max assets per push batch (default: {_BATCH_SIZE})",
+    )
+    args = parser.parse_args()
+
+    missing = [k for k, v in vars(args).items() if v is None and k not in ("output_file", "push_result_file", "batch_size")]
+    if missing:
+        parser.error(f"Missing required arguments/env vars: {missing}")
+
+    # Step 1: Collect
+    collect(
+        project_id=args.project_id,
+        output_file=args.output_file,
+    )
+
+    # Step 2: Push
+    push(
+        input_file=args.output_file,
+        resource_uuid=args.resource_uuid,
+        key_id=args.key_id,
+        key_token=args.key_token,
+        batch_size=args.batch_size,
+        output_file=args.push_result_file,
+    )
+
+
+if __name__ == "__main__":
+    main()

package/bundled-skills/monte-carlo-push-ingestion/scripts/templates/bigquery/collect_and_push_query_logs.py

@@ -0,0 +1,70 @@
+"""
+BigQuery — Query Log Collection and Push (combined)
+=====================================================
+Imports ``collect()`` from ``collect_query_logs`` and ``push()`` from
+``push_query_logs``, runs both in sequence.
+
+Substitution points (search for "← SUBSTITUTE"):
+  - BIGQUERY_PROJECT_ID   : GCP project ID to collect query logs from
+  - GOOGLE_APPLICATION_CREDENTIALS : path to service-account JSON key file
+  - LOOKBACK_HOURS        : how many hours back to collect (default 25, skip last 1 h)
+  - MCD_INGEST_ID / MCD_INGEST_TOKEN : Monte Carlo API credentials
+  - MCD_RESOURCE_UUID      : UUID of the BigQuery connection in Monte Carlo
+
+Prerequisites:
+  pip install google-cloud-bigquery pycarlo
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+
+from collect_query_logs import collect, LOOKBACK_HOURS, LOOKBACK_LAG_HOURS
+from push_query_logs import push, _BATCH_SIZE
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Push BigQuery query logs to Monte Carlo")
+    parser.add_argument("--project-id", default=os.getenv("BIGQUERY_PROJECT_ID"))  # ← SUBSTITUTE
+    parser.add_argument("--resource-uuid", default=os.getenv("MCD_RESOURCE_UUID"))
+    parser.add_argument("--key-id", default=os.getenv("MCD_INGEST_ID"))
+    parser.add_argument("--key-token", default=os.getenv("MCD_INGEST_TOKEN"))
+    parser.add_argument("--lookback-hours", type=int, default=LOOKBACK_HOURS)
+    parser.add_argument("--lookback-lag-hours", type=int, default=LOOKBACK_LAG_HOURS)
+    parser.add_argument("--output-file", default="query_logs_output.json")
+    parser.add_argument("--push-result-file", default="query_logs_push_result.json")
+    parser.add_argument(
+        "--batch-size",
+        type=int,
+        default=_BATCH_SIZE,
+        help=f"Max entries per push batch (default: {_BATCH_SIZE})",
+    )
+    args = parser.parse_args()
+
+    required = ["project_id", "resource_uuid", "key_id", "key_token"]
+    missing = [k for k in required if getattr(args, k) is None]
+    if missing:
+        parser.error(f"Missing required arguments/env vars: {missing}")
+
+    # Step 1: Collect
+    collect(
+        project_id=args.project_id,
+        lookback_hours=args.lookback_hours,
+        lookback_lag_hours=args.lookback_lag_hours,
+        output_file=args.output_file,
+    )
+
+    # Step 2: Push
+    push(
+        input_file=args.output_file,
+        resource_uuid=args.resource_uuid,
+        key_id=args.key_id,
+        key_token=args.key_token,
+        batch_size=args.batch_size,
+        output_file=args.push_result_file,
+    )
+
+
+if __name__ == "__main__":
+    main()