opencode-skills-collection 2.0.0 → 2.0.3

package/bundled-skills/monte-carlo-push-ingestion/scripts/templates/databricks/push_metadata.py

@@ -0,0 +1,178 @@
+"""
+Databricks — Metadata Push (push-only)
+========================================
+Reads a JSON manifest file produced by collect_metadata.py and pushes the assets
+to Monte Carlo via the push ingestion API, with configurable batching to keep
+compressed payloads under 1 MB.
+
+Substitution points (search for "← SUBSTITUTE"):
+  - MCD_INGEST_ID / MCD_INGEST_TOKEN : Monte Carlo API credentials
+  - MCD_RESOURCE_UUID      : UUID of the Databricks connection in Monte Carlo
+  - PUSH_BATCH_SIZE       : number of assets per API call (default 500)
+
+Prerequisites:
+  pip install pycarlo
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import os
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime, timezone
+from typing import Any
+
+from pycarlo.core import Client, Session
+from pycarlo.features.ingestion import IngestionService
+from pycarlo.features.ingestion.models import (
+    AssetField,
+    AssetFreshness,
+    AssetMetadata,
+    AssetVolume,
+    RelationalAsset,
+)
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+log = logging.getLogger(__name__)
+
+RESOURCE_TYPE = "databricks"
+DEFAULT_BATCH_SIZE = 500  # ← SUBSTITUTE: conservative default to stay under 1 MB compressed
+
+
+def _asset_from_dict(d: dict[str, Any]) -> RelationalAsset:
+    """Reconstruct a RelationalAsset from a manifest dict."""
+    fields = [
+        AssetField(
+            name=f["name"],
+            type=f.get("type"),
+            description=f.get("description"),
+        )
+        for f in d.get("fields", [])
+    ]
+
+    volume = None
+    if d.get("row_count") is not None or d.get("byte_count") is not None:
+        volume = AssetVolume(row_count=d.get("row_count"), byte_count=d.get("byte_count"))
+
+    freshness = None
+    if d.get("last_updated") is not None:
+        freshness = AssetFreshness(last_update_time=d.get("last_updated"))
+
+    return RelationalAsset(
+        type=d.get("asset_type", "TABLE"),
+        metadata=AssetMetadata(
+            name=d["asset_name"],
+            database=d["database"],    # ← SUBSTITUTE: use catalog as database
+            schema=d["schema"],
+            description=d.get("description"),
+        ),
+        fields=fields,
+        volume=volume,
+        freshness=freshness,
+    )
+
+
+def push(
+    manifest_path: str,
+    resource_uuid: str,
+    key_id: str,
+    key_token: str,
+    batch_size: int = DEFAULT_BATCH_SIZE,
+) -> dict[str, Any]:
+    """Read a collect manifest and push assets to Monte Carlo in batches.
+
+    Returns a summary dict with invocation IDs and counts.
+    """
+    with open(manifest_path) as fh:
+        manifest = json.load(fh)
+
+    asset_dicts: list[dict[str, Any]] = manifest["assets"]
+    assets = [_asset_from_dict(d) for d in asset_dicts]
+    log.info("Loaded %d assets from %s", len(assets), manifest_path)
+
+    # Split into batches
+    batches = []
+    for i in range(0, max(len(assets), 1), batch_size):
+        batches.append(assets[i : i + batch_size])
+    total_batches = len(batches)
+
+    def _push_batch(batch: list, batch_num: int) -> str | None:
+        """Push a single batch using a dedicated Session (thread-safe)."""
+        client = Client(session=Session(mcd_id=key_id, mcd_token=key_token, scope="Ingestion"))
+        service = IngestionService(mc_client=client)
+        result = service.send_metadata(
+            resource_uuid=resource_uuid,
+            resource_type=RESOURCE_TYPE,
+            events=batch,
+        )
+        invocation_id = service.extract_invocation_id(result)
+        log.info("Pushed batch %d/%d (%d assets) — invocation_id=%s", batch_num, total_batches, len(batch), invocation_id)
+        return invocation_id
+
+    # Push batches in parallel (each thread gets its own pycarlo Session)
+    max_workers = min(4, total_batches)
+    invocation_ids: list[str | None] = [None] * total_batches
+
+    with ThreadPoolExecutor(max_workers=max_workers) as pool:
+        futures = {
+            pool.submit(_push_batch, batch, i + 1): i
+            for i, batch in enumerate(batches)
+        }
+        for future in as_completed(futures):
+            idx = futures[future]
+            try:
+                invocation_ids[idx] = future.result()
+            except Exception as exc:
+                log.error("ERROR pushing batch %d: %s", idx + 1, exc)
+                raise
+
+    log.info("All %d batches pushed (%d workers)", total_batches, max_workers)
+
+    pushed_at = datetime.now(timezone.utc).isoformat()
+    summary = {
+        "resource_uuid": resource_uuid,
+        "resource_type": RESOURCE_TYPE,
+        "invocation_ids": invocation_ids,
+        "pushed_at": pushed_at,
+        "asset_count": len(assets),
+        "batch_count": total_batches,
+        "batch_size": batch_size,
+        "catalog": manifest.get("catalog"),
+    }
+
+    # Write push result alongside the collect manifest
+    push_manifest_path = manifest_path.replace(".json", "_push_result.json")
+    with open(push_manifest_path, "w") as fh:
+        json.dump(summary, fh, indent=2)
+    log.info("Push result written to %s", push_manifest_path)
+
+    return summary
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Push Databricks metadata to Monte Carlo from manifest")
+    parser.add_argument("--manifest", default="manifest_metadata.json")
+    parser.add_argument("--resource-uuid", default=os.getenv("MCD_RESOURCE_UUID"))
+    parser.add_argument("--key-id", default=os.getenv("MCD_INGEST_ID"))
+    parser.add_argument("--key-token", default=os.getenv("MCD_INGEST_TOKEN"))
+    parser.add_argument("--batch-size", type=int, default=DEFAULT_BATCH_SIZE)
+    args = parser.parse_args()
+
+    required = ["resource_uuid", "key_id", "key_token"]
+    missing = [k for k in required if getattr(args, k) is None]
+    if missing:
+        parser.error(f"Missing required arguments/env vars: {missing}")
+
+    push(
+        manifest_path=args.manifest,
+        resource_uuid=args.resource_uuid,
+        key_id=args.key_id,
+        key_token=args.key_token,
+        batch_size=args.batch_size,
+    )
+
+
+if __name__ == "__main__":
+    main()

package/bundled-skills/monte-carlo-push-ingestion/scripts/templates/databricks/push_query_logs.py

@@ -0,0 +1,200 @@
+"""
+Databricks — Query Log Push (push-only)
+=========================================
+Reads a JSON manifest file produced by collect_query_logs.py and pushes the query
+log entries to Monte Carlo via the push ingestion API, with configurable batching
+to keep compressed payloads under 1 MB.
+
+Substitution points (search for "← SUBSTITUTE"):
+  - MCD_INGEST_ID / MCD_INGEST_TOKEN : Monte Carlo API credentials
+  - MCD_RESOURCE_UUID      : UUID of the Databricks connection in Monte Carlo
+  - PUSH_BATCH_SIZE       : number of entries per API call (default 100)
+
+Prerequisites:
+  pip install pycarlo
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import os
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime, timezone
+from typing import Any
+
+from dateutil.parser import isoparse
+from pycarlo.core import Client, Session
+from pycarlo.features.ingestion import IngestionService
+from pycarlo.features.ingestion.models import QueryLogEntry
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+log = logging.getLogger(__name__)
+
+LOG_TYPE = "databricks"
+DEFAULT_BATCH_SIZE = 100  # ← SUBSTITUTE: conservative default to stay under 1 MB compressed
+
+# Truncate query_text longer than this to prevent 413 errors.
+# Some SQL statements (e.g., generated by BI tools) can be 100KB+ and blow up
+# compressed payloads even at small batch sizes.
+_MAX_QUERY_TEXT_LEN = 10_000
+
+
+def _build_query_log_entries(entry_dicts: list[dict[str, Any]]) -> list[QueryLogEntry]:
+    """Convert manifest query dicts into QueryLogEntry objects."""
+    entries = []
+    truncated = 0
+    for d in entry_dicts:
+        query_text = d.get("query_text") or ""
+
+        # Truncate very long SQL to prevent 413 Request Too Large
+        if len(query_text) > _MAX_QUERY_TEXT_LEN:
+            query_text = query_text[:_MAX_QUERY_TEXT_LEN] + "... [TRUNCATED]"
+            truncated += 1
+
+        extra = {}
+        if d.get("total_task_duration_ms") is not None:
+            extra["total_task_duration_ms"] = d["total_task_duration_ms"]
+        if d.get("read_rows") is not None:
+            extra["read_rows"] = d["read_rows"]
+        if d.get("read_bytes") is not None:
+            extra["read_bytes"] = d["read_bytes"]
+
+        start_time = d.get("start_time")
+        end_time = d.get("end_time")
+
+        entries.append(
+            QueryLogEntry(
+                query_id=d.get("query_id"),
+                query_text=query_text,
+                start_time=isoparse(start_time) if start_time else None,
+                end_time=isoparse(end_time) if end_time else None,
+                user=d.get("user"),
+                returned_rows=d.get("returned_rows"),
+                extra=extra or None,
+            )
+        )
+    if truncated:
+        log.info("Truncated %d query text(s) exceeding %d chars", truncated, _MAX_QUERY_TEXT_LEN)
+    return entries
+
+
+def push(
+    manifest_path: str,
+    resource_uuid: str,
+    key_id: str,
+    key_token: str,
+    batch_size: int = DEFAULT_BATCH_SIZE,
+) -> dict[str, Any]:
+    """Read a collect manifest and push query log entries to Monte Carlo in batches.
+
+    Returns a summary dict with invocation IDs and counts.
+    """
+    with open(manifest_path) as fh:
+        manifest = json.load(fh)
+
+    entry_dicts: list[dict[str, Any]] = manifest["entries"]
+    entries = _build_query_log_entries(entry_dicts)
+    log.info("Loaded %d query log entries from %s", len(entries), manifest_path)
+
+    if not entries:
+        log.info("No query log entries to push.")
+        summary = {
+            "resource_uuid": resource_uuid,
+            "log_type": LOG_TYPE,
+            "invocation_ids": [],
+            "pushed_at": datetime.now(timezone.utc).isoformat(),
+            "query_log_count": 0,
+            "batch_count": 0,
+            "batch_size": batch_size,
+        }
+        push_manifest_path = manifest_path.replace(".json", "_push_result.json")
+        with open(push_manifest_path, "w") as fh:
+            json.dump(summary, fh, indent=2)
+        return summary
+
+    # Split into batches
+    batches = []
+    for i in range(0, len(entries), batch_size):
+        batches.append(entries[i : i + batch_size])
+    total_batches = len(batches)
+
+    def _push_batch(batch: list, batch_num: int) -> str | None:
+        """Push a single batch using a dedicated Session (thread-safe)."""
+        client = Client(session=Session(mcd_id=key_id, mcd_token=key_token, scope="Ingestion"))
+        service = IngestionService(mc_client=client)
+        result = service.send_query_logs(
+            resource_uuid=resource_uuid,
+            log_type=LOG_TYPE,
+            events=batch,
+        )
+        invocation_id = service.extract_invocation_id(result)
+        log.info("Pushed batch %d/%d (%d entries) — invocation_id=%s", batch_num, total_batches, len(batch), invocation_id)
+        return invocation_id
+
+    # Push batches in parallel (each thread gets its own pycarlo Session)
+    max_workers = min(4, total_batches)
+    invocation_ids: list[str | None] = [None] * total_batches
+
+    with ThreadPoolExecutor(max_workers=max_workers) as pool:
+        futures = {
+            pool.submit(_push_batch, batch, i + 1): i
+            for i, batch in enumerate(batches)
+        }
+        for future in as_completed(futures):
+            idx = futures[future]
+            try:
+                invocation_ids[idx] = future.result()
+            except Exception as exc:
+                log.error("ERROR pushing batch %d: %s", idx + 1, exc)
+                raise
+
+    log.info("All %d batches pushed (%d workers)", total_batches, max_workers)
+
+    pushed_at = datetime.now(timezone.utc).isoformat()
+    summary = {
+        "resource_uuid": resource_uuid,
+        "log_type": LOG_TYPE,
+        "invocation_ids": invocation_ids,
+        "pushed_at": pushed_at,
+        "query_log_count": len(entries),
+        "batch_count": total_batches,
+        "batch_size": batch_size,
+        "lookback_hours": manifest.get("lookback_hours"),
+        "lookback_lag_hours": manifest.get("lookback_lag_hours"),
+    }
+
+    push_manifest_path = manifest_path.replace(".json", "_push_result.json")
+    with open(push_manifest_path, "w") as fh:
+        json.dump(summary, fh, indent=2)
+    log.info("Push result written to %s", push_manifest_path)
+
+    return summary
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Push Databricks query logs to Monte Carlo from manifest")
+    parser.add_argument("--manifest", default="manifest_query_logs.json")
+    parser.add_argument("--resource-uuid", default=os.getenv("MCD_RESOURCE_UUID"))
+    parser.add_argument("--key-id", default=os.getenv("MCD_INGEST_ID"))
+    parser.add_argument("--key-token", default=os.getenv("MCD_INGEST_TOKEN"))
+    parser.add_argument("--batch-size", type=int, default=DEFAULT_BATCH_SIZE)
+    args = parser.parse_args()
+
+    required = ["resource_uuid", "key_id", "key_token"]
+    missing = [k for k in required if getattr(args, k) is None]
+    if missing:
+        parser.error(f"Missing required arguments/env vars: {missing}")
+
+    push(
+        manifest_path=args.manifest,
+        resource_uuid=args.resource_uuid,
+        key_id=args.key_id,
+        key_token=args.key_token,
+        batch_size=args.batch_size,
+    )
+
+
+if __name__ == "__main__":
+    main()

package/bundled-skills/monte-carlo-push-ingestion/scripts/templates/hive/collect_and_push_lineage.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+"""
+Extract Hive lineage from a local log file and push it to Monte Carlo in one step.
+
+Thin wrapper that calls ``collect()`` from ``collect_lineage`` followed by
+``push()`` from ``push_lineage``, then writes the final manifest (with
+``resource_uuid`` and ``invocation_id``) to ``--output-file``.
+
+Substitution points
+-------------------
+- MCD_INGEST_ID    (env) / --key-id        (CLI) : Monte Carlo ingestion key ID
+- MCD_INGEST_TOKEN (env) / --key-token      (CLI) : Monte Carlo ingestion key token
+- MCD_RESOURCE_UUID    (env) / --resource-uuid  (CLI) : MC resource UUID for this connection
+- --log-file                                         : path to local HiveServer2 log
+
+Prerequisites
+-------------
+    pip install pycarlo python-dotenv
+
+Usage (table-level):
+    python collect_and_push_lineage.py \\
+        --key-id  <MCD_INGEST_ID> \\
+        --key-token <MCD_INGEST_TOKEN> \\
+        --resource-uuid <MCD_RESOURCE_UUID> \\
+        --log-file /tmp/root/hive.log
+
+Usage (column-level):
+    python collect_and_push_lineage.py ... --column-lineage
+"""
+
+import argparse
+import json
+import os
+
+from collect_lineage import collect
+from push_lineage import DEFAULT_BATCH_SIZE, DEFAULT_TIMEOUT_SECONDS, push
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Extract Hive lineage from a local log file and push to Monte Carlo",
+    )
+    # Collect args
+    parser.add_argument(
+        "--log-file",
+        default="/tmp/root/hive.log",
+        help="Path to local HiveServer2 log file (default: /tmp/root/hive.log)",  # ← SUBSTITUTE: your log path
+    )
+    # Push / MC args
+    parser.add_argument(
+        "--key-id",
+        default=os.environ.get("MCD_INGEST_ID"),
+        help="Monte Carlo ingestion key ID (env: MCD_INGEST_ID)",
+    )
+    parser.add_argument(
+        "--key-token",
+        default=os.environ.get("MCD_INGEST_TOKEN"),
+        help="Monte Carlo ingestion key token (env: MCD_INGEST_TOKEN)",
+    )
+    parser.add_argument(
+        "--resource-uuid",
+        default=os.environ.get("MCD_RESOURCE_UUID"),
+        help="Monte Carlo resource UUID for this Hive connection (env: MCD_RESOURCE_UUID)",
+    )
+    parser.add_argument(
+        "--column-lineage",
+        action="store_true",
+        help="Push column-level lineage instead of table-level",
+    )
+    parser.add_argument(
+        "--output-file",
+        default="lineage_output.json",
+        help="Path to write the lineage manifest (default: lineage_output.json)",
+    )
+    parser.add_argument(
+        "--batch-size",
+        type=int,
+        default=DEFAULT_BATCH_SIZE,
+        metavar="N",
+        help=f"Max events per POST (default: {DEFAULT_BATCH_SIZE})",
+    )
+    parser.add_argument(
+        "--timeout",
+        type=int,
+        default=DEFAULT_TIMEOUT_SECONDS,
+        metavar="SEC",
+        help=f"HTTP timeout per request in seconds (default: {DEFAULT_TIMEOUT_SECONDS})",
+    )
+    args = parser.parse_args()
+
+    if not args.key_id or not args.key_token:
+        parser.error("--key-id and --key-token are required (or set MCD_INGEST_ID / MCD_INGEST_TOKEN)")
+    if not args.resource_uuid:
+        parser.error("--resource-uuid is required (or set MCD_RESOURCE_UUID)")
+
+    manifest = collect(log_file=args.log_file)
+
+    if not manifest["edges"]:
+        print("No lineage edges detected — no CTAS or INSERT INTO ... SELECT patterns found.")
+        return
+
+    push(
+        manifest=manifest,
+        resource_uuid=args.resource_uuid,
+        key_id=args.key_id,
+        key_token=args.key_token,
+        column_lineage=args.column_lineage,
+        batch_size=args.batch_size,
+        timeout_seconds=args.timeout,
+    )
+
+    with open(args.output_file, "w") as fh:
+        json.dump(manifest, fh, indent=2)
+    print(f"Lineage manifest written to {args.output_file}")
+    print("Done.")
+
+
+if __name__ == "__main__":
+    main()

package/bundled-skills/monte-carlo-push-ingestion/scripts/templates/hive/collect_and_push_metadata.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+"""
+Collect Hive table metadata and push it to Monte Carlo in one step.
+
+Thin wrapper that calls ``collect()`` from ``collect_metadata`` followed by
+``push()`` from ``push_metadata``, then writes the final manifest (with
+``resource_uuid`` and ``invocation_id``) to ``--output-file``.
+
+Substitution points
+-------------------
+- HIVE_HOST           (env) / --hive-host      (CLI) : HiveServer2 hostname
+- MCD_INGEST_ID    (env) / --key-id         (CLI) : Monte Carlo ingestion key ID
+- MCD_INGEST_TOKEN (env) / --key-token      (CLI) : Monte Carlo ingestion key token
+- MCD_RESOURCE_UUID    (env) / --resource-uuid  (CLI) : MC resource UUID for this connection
+
+Prerequisites
+-------------
+    pip install pycarlo pyhive python-dotenv
+
+Usage
+-----
+    python collect_and_push_metadata.py \\
+        --key-id  <MCD_INGEST_ID> \\
+        --key-token <MCD_INGEST_TOKEN> \\
+        --resource-uuid <MCD_RESOURCE_UUID> \\
+        --hive-host <HIVESERVER2_HOSTNAME>
+"""
+
+import argparse
+import json
+import os
+
+from collect_metadata import collect
+from push_metadata import DEFAULT_BATCH_SIZE, DEFAULT_TIMEOUT_SECONDS, push
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Collect Hive table metadata and push to Monte Carlo",
+    )
+    # Hive / collect args
+    parser.add_argument(
+        "--hive-host",
+        default=os.environ.get("HIVE_HOST"),
+        help="HiveServer2 hostname (env: HIVE_HOST)",  # ← SUBSTITUTE: your EMR master DNS or Hive host
+    )
+    parser.add_argument(
+        "--hive-port",
+        type=int,
+        default=10000,
+        help="HiveServer2 port (default: 10000)",  # ← SUBSTITUTE if your cluster uses a non-standard port
+    )
+    # Push / MC args
+    parser.add_argument(
+        "--key-id",
+        default=os.environ.get("MCD_INGEST_ID"),
+        help="Monte Carlo ingestion key ID (env: MCD_INGEST_ID)",  # ← SUBSTITUTE env var name if different
+    )
+    parser.add_argument(
+        "--key-token",
+        default=os.environ.get("MCD_INGEST_TOKEN"),
+        help="Monte Carlo ingestion key token (env: MCD_INGEST_TOKEN)",  # ← SUBSTITUTE env var name if different
+    )
+    parser.add_argument(
+        "--resource-uuid",
+        default=os.environ.get("MCD_RESOURCE_UUID"),
+        required=False,
+        help="Monte Carlo resource UUID for this Hive connection (env: MCD_RESOURCE_UUID)",
+    )
+    parser.add_argument(
+        "--output-file",
+        default="metadata_output.json",
+        help="Path to write the output manifest (default: metadata_output.json)",
+    )
+    parser.add_argument(
+        "--batch-size",
+        type=int,
+        default=DEFAULT_BATCH_SIZE,
+        metavar="N",
+        help=f"Max assets per POST (default: {DEFAULT_BATCH_SIZE})",
+    )
+    parser.add_argument(
+        "--timeout",
+        type=int,
+        default=DEFAULT_TIMEOUT_SECONDS,
+        metavar="SEC",
+        help=f"HTTP timeout per request in seconds (default: {DEFAULT_TIMEOUT_SECONDS})",
+    )
+    args = parser.parse_args()
+
+    if not args.hive_host:
+        parser.error("--hive-host is required (or set HIVE_HOST)")
+    if not args.key_id or not args.key_token:
+        parser.error("--key-id and --key-token are required (or set MCD_INGEST_ID / MCD_INGEST_TOKEN)")
+    if not args.resource_uuid:
+        parser.error("--resource-uuid is required (or set MCD_RESOURCE_UUID)")
+
+    manifest = collect(
+        hive_host=args.hive_host,
+        hive_port=args.hive_port,
+    )
+
+    push(
+        manifest=manifest,
+        resource_uuid=args.resource_uuid,
+        key_id=args.key_id,
+        key_token=args.key_token,
+        batch_size=args.batch_size,
+        timeout_seconds=args.timeout,
+    )
+
+    with open(args.output_file, "w") as fh:
+        json.dump(manifest, fh, indent=2)
+    print(f"Manifest written to {args.output_file}")
+    print("Done.")
+
+
+if __name__ == "__main__":
+    main()