opencode-skills-collection 2.0.0 → 2.0.3

package/bundled-skills/monte-carlo-push-ingestion/scripts/templates/bigquery-iceberg/collect_and_push_query_logs.py

@@ -0,0 +1,64 @@
+"""
+BigQuery Iceberg — Query Log Collect & Push (combined)
+=====================================================
+Convenience wrapper that runs collect_query_logs.collect() followed by
+push_query_logs.push() in a single invocation.
+
+Prerequisites:
+  pip install google-cloud-bigquery pycarlo>=0.12.251 python-dateutil>=2.8.0
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+
+from collect_query_logs import LOOKBACK_HOURS, LOOKBACK_LAG_HOURS, collect
+from push_query_logs import push
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Collect BigQuery query logs and push to Monte Carlo",
+    )
+    # Collection args
+    parser.add_argument("--project-id", default=os.getenv("BIGQUERY_PROJECT_ID"))
+    parser.add_argument("--lookback-hours", type=int, default=LOOKBACK_HOURS)
+    parser.add_argument("--lookback-lag-hours", type=int, default=LOOKBACK_LAG_HOURS)
+    parser.add_argument("--manifest-file", default="query_logs_output.json")
+
+    # Push args
+    parser.add_argument("--resource-uuid", default=os.getenv("MCD_RESOURCE_UUID"))
+    parser.add_argument("--key-id", default=os.getenv("MCD_INGEST_ID"))
+    parser.add_argument("--key-token", default=os.getenv("MCD_INGEST_TOKEN"))
+    parser.add_argument("--batch-size", type=int, default=100)
+    parser.add_argument("--push-result-file", default="query_logs_push_result.json")
+
+    args = parser.parse_args()
+
+    if not args.project_id:
+        parser.error("--project-id or BIGQUERY_PROJECT_ID env var is required")
+    required_push = ["resource_uuid", "key_id", "key_token"]
+    missing = [k for k in required_push if getattr(args, k) is None]
+    if missing:
+        parser.error(f"Missing required push arguments/env vars: {missing}")
+
+    collect(
+        project_id=args.project_id,
+        lookback_hours=args.lookback_hours,
+        lookback_lag_hours=args.lookback_lag_hours,
+        output_file=args.manifest_file,
+    )
+
+    push(
+        input_file=args.manifest_file,
+        resource_uuid=args.resource_uuid,
+        key_id=args.key_id,
+        key_token=args.key_token,
+        batch_size=args.batch_size,
+        output_file=args.push_result_file,
+    )
+
+
+if __name__ == "__main__":
+    main()

package/bundled-skills/monte-carlo-push-ingestion/scripts/templates/bigquery-iceberg/collect_metadata.py

@@ -0,0 +1,253 @@
+"""
+BigQuery Iceberg — Metadata Collection (collect only)
+=====================================================
+Collects table schemas, row counts, byte sizes, and freshness for BigQuery
+Iceberg (BigLake-managed) tables using INFORMATION_SCHEMA.TABLE_STORAGE and
+INFORMATION_SCHEMA.COLUMNS. Standard BigQuery collection uses __TABLES__ which
+does not include Iceberg tables — this template fills that gap.
+
+Can be run standalone via CLI or imported (use the ``collect()`` function).
+
+Supports a ``--only-freshness-and-volume`` flag to skip the COLUMNS query for
+fast periodic pushes after the initial full metadata push.
+
+Substitution points (search for "← SUBSTITUTE"):
+  - BIGQUERY_PROJECT_ID                : GCP project ID to collect from
+  - GOOGLE_APPLICATION_CREDENTIALS     : path to service-account JSON key file
+  - REGION                             : BigQuery region (default "us")
+
+Prerequisites:
+  pip install google-cloud-bigquery
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import os
+from datetime import datetime, timezone
+
+from google.cloud import bigquery
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+log = logging.getLogger(__name__)
+
+RESOURCE_TYPE = "bigquery"
+
+# BigQuery type → Monte Carlo canonical type
+BQ_TYPE_MAP: dict[str, str] = {
+    "INT64": "INTEGER",
+    "INTEGER": "INTEGER",
+    "FLOAT64": "FLOAT",
+    "FLOAT": "FLOAT",
+    "BOOL": "BOOLEAN",
+    "BOOLEAN": "BOOLEAN",
+    "STRING": "VARCHAR",
+    "BYTES": "BINARY",
+    "DATE": "DATE",
+    "DATETIME": "DATETIME",
+    "TIMESTAMP": "TIMESTAMP",
+    "TIME": "TIME",
+    "NUMERIC": "DECIMAL",
+    "BIGNUMERIC": "DECIMAL",
+    "RECORD": "STRUCT",
+    "STRUCT": "STRUCT",
+    "REPEATED": "ARRAY",
+    "JSON": "JSON",
+    "GEOGRAPHY": "GEOGRAPHY",
+}
+
+
+def map_bq_type(bq_type: str) -> str:
+    base = bq_type.split("(")[0].strip().upper()
+    return BQ_TYPE_MAP.get(base, bq_type.upper())
+
+
+def _fetch_iceberg_tables(
+    client: bigquery.Client,
+    project_id: str,
+    datasets: list[str] | None = None,
+    tables: list[str] | None = None,
+) -> list[dict]:
+    """Query TABLE_STORAGE for BigLake (Iceberg) tables."""
+    conditions = [
+        "managed_table_type = 'BIGLAKE'",
+        "deleted = FALSE",
+    ]
+    if datasets:
+        ds_list = ", ".join(f"'{d}'" for d in datasets)
+        conditions.append(f"table_schema IN ({ds_list})")
+    if tables:
+        tbl_list = ", ".join(f"'{t}'" for t in tables)
+        conditions.append(f"table_name IN ({tbl_list})")
+
+    where = " AND ".join(conditions)
+    query = f"""
+        SELECT
+            table_schema,
+            table_name,
+            total_rows,
+            current_physical_bytes,
+            storage_last_modified_time,
+            creation_time
+        FROM `{project_id}.region-us`.INFORMATION_SCHEMA.TABLE_STORAGE  -- ← SUBSTITUTE: change region if needed
+        WHERE {where}
+        ORDER BY table_schema, table_name
+    """
+    log.info("Querying TABLE_STORAGE for Iceberg tables ...")
+    rows = list(client.query(query).result())
+    log.info("Found %d Iceberg table(s).", len(rows))
+    return [dict(row) for row in rows]
+
+
+def _fetch_columns(
+    client: bigquery.Client,
+    project_id: str,
+    dataset: str,
+    table_name: str,
+) -> list[dict]:
+    """Fetch column metadata for a specific table."""
+    query = f"""
+        SELECT column_name, data_type, ordinal_position, is_nullable, column_default
+        FROM `{project_id}.{dataset}.INFORMATION_SCHEMA.COLUMNS`
+        WHERE table_name = '{table_name}'
+        ORDER BY ordinal_position
+    """
+    return [
+        {
+            "name": row["column_name"],
+            "type": map_bq_type(row["data_type"]),
+        }
+        for row in client.query(query).result()
+    ]
+
+
+def _resolve_freshness(row: dict) -> str:
+    """Return the best available freshness timestamp as ISO8601.
+
+    Uses storage_last_modified_time if Google has populated it (expected
+    early April 2026). Falls back to current time with a warning.
+    """
+    if row.get("storage_last_modified_time"):
+        return row["storage_last_modified_time"].isoformat()
+
+    log.warning(
+        "storage_last_modified_time is NULL for %s.%s — "
+        "falling back to current time. Google's TABLE_STORAGE update "
+        "for Iceberg tables may not have shipped yet.",
+        row["table_schema"],
+        row["table_name"],
+    )
+    return datetime.now(timezone.utc).isoformat()
+
+
+def collect(
+    project_id: str,
+    datasets: list[str] | None = None,
+    tables: list[str] | None = None,
+    only_freshness_and_volume: bool = False,
+    output_file: str = "metadata_output.json",
+) -> dict:
+    """Collect Iceberg table metadata and write a JSON manifest.
+
+    When only_freshness_and_volume is True, skips the COLUMNS query and
+    omits fields from the manifest. Use this for periodic hourly pushes
+    after the initial full metadata push.
+    """
+    client = bigquery.Client(project=project_id)  # ← SUBSTITUTE: adjust auth if needed
+
+    if only_freshness_and_volume:
+        log.info("Running in freshness+volume only mode (skipping fields).")
+
+    iceberg_tables = _fetch_iceberg_tables(client, project_id, datasets, tables)
+    if not iceberg_tables:
+        log.warning("No Iceberg tables found matching the criteria.")
+        return {"resource_type": RESOURCE_TYPE, "assets": []}
+
+    assets: list[dict] = []
+    for row in iceberg_tables:
+        dataset = row["table_schema"]
+        name = row["table_name"]
+
+        asset = {
+            "name": name,
+            "database": project_id,
+            "schema": dataset,
+            "type": "TABLE",
+            "volume": {
+                "row_count": row["total_rows"],
+                "byte_count": row["current_physical_bytes"],
+            },
+            "freshness": {
+                "last_updated_time": _resolve_freshness(row),
+            },
+        }
+
+        if not only_freshness_and_volume:
+            asset["description"] = None
+            asset["fields"] = _fetch_columns(client, project_id, dataset, name)
+
+        assets.append(asset)
+        log.info(
+            "Collected %s.%s.%s — rows=%s, bytes=%s",
+            project_id, dataset, name,
+            row["total_rows"], row["current_physical_bytes"],
+        )
+
+    manifest = {
+        "resource_type": RESOURCE_TYPE,
+        "collected_at": datetime.now(timezone.utc).isoformat(),
+        "assets": assets,
+    }
+    with open(output_file, "w") as fh:
+        json.dump(manifest, fh, indent=2)
+    log.info("Manifest written to %s (%d assets)", output_file, len(assets))
+
+    return manifest
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Collect BigQuery Iceberg table metadata into a JSON manifest",
+    )
+    parser.add_argument(
+        "--project-id",
+        default=os.getenv("BIGQUERY_PROJECT_ID"),  # ← SUBSTITUTE
+        help="GCP project ID (or set BIGQUERY_PROJECT_ID env var)",
+    )
+    parser.add_argument(
+        "--datasets",
+        nargs="+",
+        default=None,
+        help="Limit to specific dataset(s). Omit to scan all datasets.",
+    )
+    parser.add_argument(
+        "--tables",
+        nargs="+",
+        default=None,
+        help="Limit to specific table name(s) within the datasets.",
+    )
+    parser.add_argument(
+        "--only-freshness-and-volume",
+        action="store_true",
+        help="Skip field/schema collection — only collect freshness and volume. "
+             "Use for periodic hourly pushes after the initial full metadata push.",
+    )
+    parser.add_argument("--output-file", default="metadata_output.json")
+    args = parser.parse_args()
+
+    if not args.project_id:
+        parser.error("--project-id or BIGQUERY_PROJECT_ID env var is required")
+
+    collect(
+        project_id=args.project_id,
+        datasets=args.datasets,
+        tables=args.tables,
+        only_freshness_and_volume=args.only_freshness_and_volume,
+        output_file=args.output_file,
+    )
+
+
+if __name__ == "__main__":
+    main()

package/bundled-skills/monte-carlo-push-ingestion/scripts/templates/bigquery-iceberg/collect_query_logs.py

@@ -0,0 +1,149 @@
+"""
+BigQuery Iceberg — Query Log Collection (collect only)
+======================================================
+Queries the BigQuery Jobs API for completed query jobs within a time
+window and writes a JSON manifest that can be fed to push_query_logs.py.
+
+Can be run standalone via CLI or imported (use the ``collect()`` function).
+
+Substitution points (search for "← SUBSTITUTE"):
+  - BIGQUERY_PROJECT_ID                : GCP project ID to collect from
+  - GOOGLE_APPLICATION_CREDENTIALS     : path to service-account JSON key file
+
+Prerequisites:
+  pip install google-cloud-bigquery
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import os
+from datetime import datetime, timedelta, timezone
+
+from google.cloud import bigquery
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+log = logging.getLogger(__name__)
+
+LOG_TYPE = "bigquery"
+
+LOOKBACK_HOURS: int = int(os.getenv("LOOKBACK_HOURS", "25"))
+LOOKBACK_LAG_HOURS: int = int(os.getenv("LOOKBACK_LAG_HOURS", "1"))
+MAX_JOBS: int = int(os.getenv("MAX_JOBS", "10000"))
+
+# Limit to specific statement types — empty list means collect all.
+STATEMENT_TYPE_FILTER: list[str] = []
+
+
+def _safe_isoformat(dt: datetime | None) -> str | None:
+    if dt is None:
+        return None
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=timezone.utc)
+    return dt.isoformat()
+
+
+def _collect_query_logs(
+    bq_client: bigquery.Client,
+    project_id: str,
+    start_dt: datetime,
+    end_dt: datetime,
+) -> list[dict]:
+    """Collect query logs from BigQuery job history."""
+    entries: list[dict] = []
+
+    log.info(
+        "Listing jobs for project=%s from %s to %s",
+        project_id, start_dt.isoformat(), end_dt.isoformat(),
+    )
+
+    for job in bq_client.list_jobs(
+        project=project_id,
+        all_users=True,
+        min_creation_time=start_dt,
+        max_creation_time=end_dt,
+    ):
+        sql: str = getattr(job, "query", None) or ""
+        if not sql.strip():
+            continue
+
+        statement_type: str = getattr(job, "statement_type", None) or ""
+        if STATEMENT_TYPE_FILTER and statement_type not in STATEMENT_TYPE_FILTER:
+            continue
+
+        entries.append({
+            "query_id": job.job_id,
+            "query_text": sql,
+            "start_time": _safe_isoformat(getattr(job, "created", None)),
+            "end_time": _safe_isoformat(getattr(job, "ended", None)),
+            "user": getattr(job, "user_email", None),
+            "total_bytes_billed": getattr(job, "total_bytes_billed", None),
+            "statement_type": statement_type or None,
+        })
+
+        if len(entries) >= MAX_JOBS:
+            log.warning("Reached MAX_JOBS=%d — stopping early", MAX_JOBS)
+            break
+
+    return entries
+
+
+def collect(
+    project_id: str,
+    lookback_hours: int = LOOKBACK_HOURS,
+    lookback_lag_hours: int = LOOKBACK_LAG_HOURS,
+    output_file: str = "query_logs_output.json",
+) -> dict:
+    """Collect query logs and write a JSON manifest."""
+    bq_client = bigquery.Client(project=project_id)
+
+    end_dt = datetime.now(timezone.utc) - timedelta(hours=lookback_lag_hours)
+    start_dt = end_dt - timedelta(hours=lookback_hours)
+
+    entries = _collect_query_logs(bq_client, project_id, start_dt, end_dt)
+    log.info("Collected %d query log entries.", len(entries))
+
+    manifest = {
+        "log_type": LOG_TYPE,
+        "collected_at": datetime.now(timezone.utc).isoformat(),
+        "window_start": start_dt.isoformat(),
+        "window_end": end_dt.isoformat(),
+        "query_log_count": len(entries),
+        "queries": entries,
+    }
+    with open(output_file, "w") as fh:
+        json.dump(manifest, fh, indent=2)
+    log.info("Query log manifest written to %s", output_file)
+
+    return manifest
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Collect BigQuery query logs into a JSON manifest",
+    )
+    parser.add_argument(
+        "--project-id",
+        default=os.getenv("BIGQUERY_PROJECT_ID"),
+        help="GCP project ID (or set BIGQUERY_PROJECT_ID env var)",
+    )
+    parser.add_argument("--lookback-hours", type=int, default=LOOKBACK_HOURS)
+    parser.add_argument("--lookback-lag-hours", type=int, default=LOOKBACK_LAG_HOURS)
+    parser.add_argument("--output-file", default="query_logs_output.json")
+    args = parser.parse_args()
+
+    if not args.project_id:
+        parser.error("--project-id or BIGQUERY_PROJECT_ID env var is required")
+
+    collect(
+        project_id=args.project_id,
+        lookback_hours=args.lookback_hours,
+        lookback_lag_hours=args.lookback_lag_hours,
+        output_file=args.output_file,
+    )
+
+
+if __name__ == "__main__":
+    main()

package/bundled-skills/monte-carlo-push-ingestion/scripts/templates/bigquery-iceberg/push_metadata.py

@@ -0,0 +1,190 @@
+"""
+BigQuery Iceberg — Metadata Push (push only)
+============================================
+Reads a JSON manifest produced by collect_metadata.py and pushes table
+metadata to Monte Carlo using the pycarlo SDK's IngestionService.
+
+Can be run standalone via CLI or imported (use the ``push()`` function).
+
+Substitution points (search for "← SUBSTITUTE"):
+  - MCD_INGEST_ID      : Monte Carlo Ingestion API key ID
+  - MCD_INGEST_TOKEN   : Monte Carlo Ingestion API key token
+  - MCD_RESOURCE_UUID  : Monte Carlo warehouse resource UUID
+
+Prerequisites:
+  pip install pycarlo>=0.12.251
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import os
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime, timezone
+
+from pycarlo.core import Client, Session
+from pycarlo.features.ingestion import IngestionService
+from pycarlo.features.ingestion.models import (
+    AssetField,
+    AssetFreshness,
+    AssetMetadata,
+    AssetVolume,
+    RelationalAsset,
+)
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+log = logging.getLogger(__name__)
+
+RESOURCE_TYPE = "bigquery"
+_BATCH_SIZE = 500
+
+_ENDPOINT = "https://integrations.getmontecarlo.com"
+
+
+def _asset_from_dict(d: dict) -> RelationalAsset:
+    """Reconstruct a RelationalAsset from a manifest dict entry."""
+    fields = [
+        AssetField(
+            name=f["name"],
+            type=f.get("type"),
+            description=f.get("description"),
+        )
+        for f in d.get("fields", [])
+    ]
+
+    volume = None
+    if d.get("volume"):
+        volume = AssetVolume(
+            row_count=d["volume"].get("row_count"),
+            byte_count=d["volume"].get("byte_count"),
+        )
+
+    freshness = None
+    if d.get("freshness") and d["freshness"].get("last_updated_time"):
+        freshness = AssetFreshness(
+            last_update_time=d["freshness"]["last_updated_time"],
+        )
+
+    return RelationalAsset(
+        type=d.get("type", "TABLE"),
+        metadata=AssetMetadata(
+            name=d["name"],
+            database=d["database"],
+            schema=d["schema"],
+            description=d.get("description"),
+        ),
+        fields=fields,
+        volume=volume,
+        freshness=freshness,
+    )
+
+
+def push(
+    input_file: str,
+    resource_uuid: str,
+    key_id: str,
+    key_token: str,
+    batch_size: int = _BATCH_SIZE,
+    output_file: str = "metadata_push_result.json",
+) -> dict:
+    """Read a metadata manifest and push assets to Monte Carlo in batches."""
+    endpoint = _ENDPOINT
+    log.info("Using endpoint: %s", endpoint)
+    with open(input_file) as fh:
+        manifest = json.load(fh)
+
+    asset_dicts = manifest.get("assets", [])
+    resource_type = manifest.get("resource_type", RESOURCE_TYPE)
+    assets = [_asset_from_dict(d) for d in asset_dicts]
+    log.info("Loaded %d asset(s) from %s", len(assets), input_file)
+
+    batches = [assets[i : i + batch_size] for i in range(0, max(len(assets), 1), batch_size)]
+    total_batches = len(batches)
+
+    def _push_batch(batch: list[RelationalAsset], batch_num: int) -> str | None:
+        client = Client(session=Session(
+            mcd_id=key_id, mcd_token=key_token, scope="Ingestion", endpoint=endpoint,
+        ))
+        service = IngestionService(mc_client=client)
+        result = service.send_metadata(
+            resource_uuid=resource_uuid,
+            resource_type=resource_type,
+            events=batch,
+        )
+        invocation_id = service.extract_invocation_id(result)
+        log.info(
+            "Pushed batch %d/%d (%d assets) — invocation_id=%s",
+            batch_num, total_batches, len(batch), invocation_id,
+        )
+        return invocation_id
+
+    max_workers = min(4, total_batches)
+    invocation_ids: list[str | None] = [None] * total_batches
+
+    with ThreadPoolExecutor(max_workers=max_workers) as pool:
+        futures = {
+            pool.submit(_push_batch, batch, i + 1): i
+            for i, batch in enumerate(batches)
+        }
+        for future in as_completed(futures):
+            idx = futures[future]
+            try:
+                invocation_ids[idx] = future.result()
+            except Exception as exc:
+                log.error("ERROR pushing batch %d: %s", idx + 1, exc)
+                raise
+
+    log.info("All %d batch(es) pushed.", total_batches)
+
+    push_result = {
+        "resource_uuid": resource_uuid,
+        "resource_type": resource_type,
+        "invocation_ids": invocation_ids,
+        "pushed_at": datetime.now(timezone.utc).isoformat(),
+        "total_assets": len(assets),
+        "batch_count": total_batches,
+        "batch_size": batch_size,
+    }
+    with open(output_file, "w") as fh:
+        json.dump(push_result, fh, indent=2)
+    log.info("Push result written to %s", output_file)
+
+    return push_result
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Push BigQuery Iceberg metadata from a manifest to Monte Carlo",
+    )
+    parser.add_argument("--resource-uuid", default=os.getenv("MCD_RESOURCE_UUID"))
+    parser.add_argument("--key-id", default=os.getenv("MCD_INGEST_ID"))
+    parser.add_argument("--key-token", default=os.getenv("MCD_INGEST_TOKEN"))
+    parser.add_argument("--input-file", default="metadata_output.json")
+    parser.add_argument("--output-file", default="metadata_push_result.json")
+    parser.add_argument(
+        "--batch-size",
+        type=int,
+        default=_BATCH_SIZE,
+        help=f"Max assets per push batch (default: {_BATCH_SIZE})",
+    )
+    args = parser.parse_args()
+
+    required = ["resource_uuid", "key_id", "key_token"]
+    missing = [k for k in required if getattr(args, k) is None]
+    if missing:
+        parser.error(f"Missing required arguments/env vars: {missing}")
+
+    push(
+        input_file=args.input_file,
+        resource_uuid=args.resource_uuid,
+        key_id=args.key_id,
+        key_token=args.key_token,
+        batch_size=args.batch_size,
+        output_file=args.output_file,
+    )
+
+
+if __name__ == "__main__":
+    main()