opencode-skills-collection 3.1.0 → 3.1.1

package/bundled-skills/monte-carlo-push-ingestion/scripts/templates/bigquery-iceberg/collect_and_push_query_logs.py

@@ -1,64 +0,0 @@
-"""
-BigQuery Iceberg — Query Log Collect & Push (combined)
-=====================================================
-Convenience wrapper that runs collect_query_logs.collect() followed by
-push_query_logs.push() in a single invocation.
-
-Prerequisites:
-  pip install google-cloud-bigquery pycarlo>=0.12.251 python-dateutil>=2.8.0
-"""
-
-from __future__ import annotations
-
-import argparse
-import os
-
-from collect_query_logs import LOOKBACK_HOURS, LOOKBACK_LAG_HOURS, collect
-from push_query_logs import push
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(
-        description="Collect BigQuery query logs and push to Monte Carlo",
-    )
-    # Collection args
-    parser.add_argument("--project-id", default=os.getenv("BIGQUERY_PROJECT_ID"))
-    parser.add_argument("--lookback-hours", type=int, default=LOOKBACK_HOURS)
-    parser.add_argument("--lookback-lag-hours", type=int, default=LOOKBACK_LAG_HOURS)
-    parser.add_argument("--manifest-file", default="query_logs_output.json")
-
-    # Push args
-    parser.add_argument("--resource-uuid", default=os.getenv("MCD_RESOURCE_UUID"))
-    parser.add_argument("--key-id", default=os.getenv("MCD_INGEST_ID"))
-    parser.add_argument("--key-token", default=os.getenv("MCD_INGEST_TOKEN"))
-    parser.add_argument("--batch-size", type=int, default=100)
-    parser.add_argument("--push-result-file", default="query_logs_push_result.json")
-
-    args = parser.parse_args()
-
-    if not args.project_id:
-        parser.error("--project-id or BIGQUERY_PROJECT_ID env var is required")
-    required_push = ["resource_uuid", "key_id", "key_token"]
-    missing = [k for k in required_push if getattr(args, k) is None]
-    if missing:
-        parser.error(f"Missing required push arguments/env vars: {missing}")
-
-    collect(
-        project_id=args.project_id,
-        lookback_hours=args.lookback_hours,
-        lookback_lag_hours=args.lookback_lag_hours,
-        output_file=args.manifest_file,
-    )
-
-    push(
-        input_file=args.manifest_file,
-        resource_uuid=args.resource_uuid,
-        key_id=args.key_id,
-        key_token=args.key_token,
-        batch_size=args.batch_size,
-        output_file=args.push_result_file,
-    )
-
-
-if __name__ == "__main__":
-    main()

package/bundled-skills/monte-carlo-push-ingestion/scripts/templates/bigquery-iceberg/collect_metadata.py

@@ -1,253 +0,0 @@
-"""
-BigQuery Iceberg — Metadata Collection (collect only)
-=====================================================
-Collects table schemas, row counts, byte sizes, and freshness for BigQuery
-Iceberg (BigLake-managed) tables using INFORMATION_SCHEMA.TABLE_STORAGE and
-INFORMATION_SCHEMA.COLUMNS. Standard BigQuery collection uses __TABLES__ which
-does not include Iceberg tables — this template fills that gap.
-
-Can be run standalone via CLI or imported (use the ``collect()`` function).
-
-Supports a ``--only-freshness-and-volume`` flag to skip the COLUMNS query for
-fast periodic pushes after the initial full metadata push.
-
-Substitution points (search for "← SUBSTITUTE"):
-  - BIGQUERY_PROJECT_ID                : GCP project ID to collect from
-  - GOOGLE_APPLICATION_CREDENTIALS     : path to service-account JSON key file
-  - REGION                             : BigQuery region (default "us")
-
-Prerequisites:
-  pip install google-cloud-bigquery
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import logging
-import os
-from datetime import datetime, timezone
-
-from google.cloud import bigquery
-
-logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
-log = logging.getLogger(__name__)
-
-RESOURCE_TYPE = "bigquery"
-
-# BigQuery type → Monte Carlo canonical type
-BQ_TYPE_MAP: dict[str, str] = {
-    "INT64": "INTEGER",
-    "INTEGER": "INTEGER",
-    "FLOAT64": "FLOAT",
-    "FLOAT": "FLOAT",
-    "BOOL": "BOOLEAN",
-    "BOOLEAN": "BOOLEAN",
-    "STRING": "VARCHAR",
-    "BYTES": "BINARY",
-    "DATE": "DATE",
-    "DATETIME": "DATETIME",
-    "TIMESTAMP": "TIMESTAMP",
-    "TIME": "TIME",
-    "NUMERIC": "DECIMAL",
-    "BIGNUMERIC": "DECIMAL",
-    "RECORD": "STRUCT",
-    "STRUCT": "STRUCT",
-    "REPEATED": "ARRAY",
-    "JSON": "JSON",
-    "GEOGRAPHY": "GEOGRAPHY",
-}
-
-
-def map_bq_type(bq_type: str) -> str:
-    base = bq_type.split("(")[0].strip().upper()
-    return BQ_TYPE_MAP.get(base, bq_type.upper())
-
-
-def _fetch_iceberg_tables(
-    client: bigquery.Client,
-    project_id: str,
-    datasets: list[str] | None = None,
-    tables: list[str] | None = None,
-) -> list[dict]:
-    """Query TABLE_STORAGE for BigLake (Iceberg) tables."""
-    conditions = [
-        "managed_table_type = 'BIGLAKE'",
-        "deleted = FALSE",
-    ]
-    if datasets:
-        ds_list = ", ".join(f"'{d}'" for d in datasets)
-        conditions.append(f"table_schema IN ({ds_list})")
-    if tables:
-        tbl_list = ", ".join(f"'{t}'" for t in tables)
-        conditions.append(f"table_name IN ({tbl_list})")
-
-    where = " AND ".join(conditions)
-    query = f"""
-        SELECT
-            table_schema,
-            table_name,
-            total_rows,
-            current_physical_bytes,
-            storage_last_modified_time,
-            creation_time
-        FROM `{project_id}.region-us`.INFORMATION_SCHEMA.TABLE_STORAGE  -- ← SUBSTITUTE: change region if needed
-        WHERE {where}
-        ORDER BY table_schema, table_name
-    """
-    log.info("Querying TABLE_STORAGE for Iceberg tables ...")
-    rows = list(client.query(query).result())
-    log.info("Found %d Iceberg table(s).", len(rows))
-    return [dict(row) for row in rows]
-
-
-def _fetch_columns(
-    client: bigquery.Client,
-    project_id: str,
-    dataset: str,
-    table_name: str,
-) -> list[dict]:
-    """Fetch column metadata for a specific table."""
-    query = f"""
-        SELECT column_name, data_type, ordinal_position, is_nullable, column_default
-        FROM `{project_id}.{dataset}.INFORMATION_SCHEMA.COLUMNS`
-        WHERE table_name = '{table_name}'
-        ORDER BY ordinal_position
-    """
-    return [
-        {
-            "name": row["column_name"],
-            "type": map_bq_type(row["data_type"]),
-        }
-        for row in client.query(query).result()
-    ]
-
-
-def _resolve_freshness(row: dict) -> str:
-    """Return the best available freshness timestamp as ISO8601.
-
-    Uses storage_last_modified_time if Google has populated it (expected
-    early April 2026). Falls back to current time with a warning.
-    """
-    if row.get("storage_last_modified_time"):
-        return row["storage_last_modified_time"].isoformat()
-
-    log.warning(
-        "storage_last_modified_time is NULL for %s.%s — "
-        "falling back to current time. Google's TABLE_STORAGE update "
-        "for Iceberg tables may not have shipped yet.",
-        row["table_schema"],
-        row["table_name"],
-    )
-    return datetime.now(timezone.utc).isoformat()
-
-
-def collect(
-    project_id: str,
-    datasets: list[str] | None = None,
-    tables: list[str] | None = None,
-    only_freshness_and_volume: bool = False,
-    output_file: str = "metadata_output.json",
-) -> dict:
-    """Collect Iceberg table metadata and write a JSON manifest.
-
-    When only_freshness_and_volume is True, skips the COLUMNS query and
-    omits fields from the manifest. Use this for periodic hourly pushes
-    after the initial full metadata push.
-    """
-    client = bigquery.Client(project=project_id)  # ← SUBSTITUTE: adjust auth if needed
-
-    if only_freshness_and_volume:
-        log.info("Running in freshness+volume only mode (skipping fields).")
-
-    iceberg_tables = _fetch_iceberg_tables(client, project_id, datasets, tables)
-    if not iceberg_tables:
-        log.warning("No Iceberg tables found matching the criteria.")
-        return {"resource_type": RESOURCE_TYPE, "assets": []}
-
-    assets: list[dict] = []
-    for row in iceberg_tables:
-        dataset = row["table_schema"]
-        name = row["table_name"]
-
-        asset = {
-            "name": name,
-            "database": project_id,
-            "schema": dataset,
-            "type": "TABLE",
-            "volume": {
-                "row_count": row["total_rows"],
-                "byte_count": row["current_physical_bytes"],
-            },
-            "freshness": {
-                "last_updated_time": _resolve_freshness(row),
-            },
-        }
-
-        if not only_freshness_and_volume:
-            asset["description"] = None
-            asset["fields"] = _fetch_columns(client, project_id, dataset, name)
-
-        assets.append(asset)
-        log.info(
-            "Collected %s.%s.%s — rows=%s, bytes=%s",
-            project_id, dataset, name,
-            row["total_rows"], row["current_physical_bytes"],
-        )
-
-    manifest = {
-        "resource_type": RESOURCE_TYPE,
-        "collected_at": datetime.now(timezone.utc).isoformat(),
-        "assets": assets,
-    }
-    with open(output_file, "w") as fh:
-        json.dump(manifest, fh, indent=2)
-    log.info("Manifest written to %s (%d assets)", output_file, len(assets))
-
-    return manifest
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(
-        description="Collect BigQuery Iceberg table metadata into a JSON manifest",
-    )
-    parser.add_argument(
-        "--project-id",
-        default=os.getenv("BIGQUERY_PROJECT_ID"),  # ← SUBSTITUTE
-        help="GCP project ID (or set BIGQUERY_PROJECT_ID env var)",
-    )
-    parser.add_argument(
-        "--datasets",
-        nargs="+",
-        default=None,
-        help="Limit to specific dataset(s). Omit to scan all datasets.",
-    )
-    parser.add_argument(
-        "--tables",
-        nargs="+",
-        default=None,
-        help="Limit to specific table name(s) within the datasets.",
-    )
-    parser.add_argument(
-        "--only-freshness-and-volume",
-        action="store_true",
-        help="Skip field/schema collection — only collect freshness and volume. "
-             "Use for periodic hourly pushes after the initial full metadata push.",
-    )
-    parser.add_argument("--output-file", default="metadata_output.json")
-    args = parser.parse_args()
-
-    if not args.project_id:
-        parser.error("--project-id or BIGQUERY_PROJECT_ID env var is required")
-
-    collect(
-        project_id=args.project_id,
-        datasets=args.datasets,
-        tables=args.tables,
-        only_freshness_and_volume=args.only_freshness_and_volume,
-        output_file=args.output_file,
-    )
-
-
-if __name__ == "__main__":
-    main()

package/bundled-skills/monte-carlo-push-ingestion/scripts/templates/bigquery-iceberg/collect_query_logs.py

@@ -1,149 +0,0 @@
-"""
-BigQuery Iceberg — Query Log Collection (collect only)
-======================================================
-Queries the BigQuery Jobs API for completed query jobs within a time
-window and writes a JSON manifest that can be fed to push_query_logs.py.
-
-Can be run standalone via CLI or imported (use the ``collect()`` function).
-
-Substitution points (search for "← SUBSTITUTE"):
-  - BIGQUERY_PROJECT_ID                : GCP project ID to collect from
-  - GOOGLE_APPLICATION_CREDENTIALS     : path to service-account JSON key file
-
-Prerequisites:
-  pip install google-cloud-bigquery
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import logging
-import os
-from datetime import datetime, timedelta, timezone
-
-from google.cloud import bigquery
-
-logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
-log = logging.getLogger(__name__)
-
-LOG_TYPE = "bigquery"
-
-LOOKBACK_HOURS: int = int(os.getenv("LOOKBACK_HOURS", "25"))
-LOOKBACK_LAG_HOURS: int = int(os.getenv("LOOKBACK_LAG_HOURS", "1"))
-MAX_JOBS: int = int(os.getenv("MAX_JOBS", "10000"))
-
-# Limit to specific statement types — empty list means collect all.
-STATEMENT_TYPE_FILTER: list[str] = []
-
-
-def _safe_isoformat(dt: datetime | None) -> str | None:
-    if dt is None:
-        return None
-    if dt.tzinfo is None:
-        dt = dt.replace(tzinfo=timezone.utc)
-    return dt.isoformat()
-
-
-def _collect_query_logs(
-    bq_client: bigquery.Client,
-    project_id: str,
-    start_dt: datetime,
-    end_dt: datetime,
-) -> list[dict]:
-    """Collect query logs from BigQuery job history."""
-    entries: list[dict] = []
-
-    log.info(
-        "Listing jobs for project=%s from %s to %s",
-        project_id, start_dt.isoformat(), end_dt.isoformat(),
-    )
-
-    for job in bq_client.list_jobs(
-        project=project_id,
-        all_users=True,
-        min_creation_time=start_dt,
-        max_creation_time=end_dt,
-    ):
-        sql: str = getattr(job, "query", None) or ""
-        if not sql.strip():
-            continue
-
-        statement_type: str = getattr(job, "statement_type", None) or ""
-        if STATEMENT_TYPE_FILTER and statement_type not in STATEMENT_TYPE_FILTER:
-            continue
-
-        entries.append({
-            "query_id": job.job_id,
-            "query_text": sql,
-            "start_time": _safe_isoformat(getattr(job, "created", None)),
-            "end_time": _safe_isoformat(getattr(job, "ended", None)),
-            "user": getattr(job, "user_email", None),
-            "total_bytes_billed": getattr(job, "total_bytes_billed", None),
-            "statement_type": statement_type or None,
-        })
-
-        if len(entries) >= MAX_JOBS:
-            log.warning("Reached MAX_JOBS=%d — stopping early", MAX_JOBS)
-            break
-
-    return entries
-
-
-def collect(
-    project_id: str,
-    lookback_hours: int = LOOKBACK_HOURS,
-    lookback_lag_hours: int = LOOKBACK_LAG_HOURS,
-    output_file: str = "query_logs_output.json",
-) -> dict:
-    """Collect query logs and write a JSON manifest."""
-    bq_client = bigquery.Client(project=project_id)
-
-    end_dt = datetime.now(timezone.utc) - timedelta(hours=lookback_lag_hours)
-    start_dt = end_dt - timedelta(hours=lookback_hours)
-
-    entries = _collect_query_logs(bq_client, project_id, start_dt, end_dt)
-    log.info("Collected %d query log entries.", len(entries))
-
-    manifest = {
-        "log_type": LOG_TYPE,
-        "collected_at": datetime.now(timezone.utc).isoformat(),
-        "window_start": start_dt.isoformat(),
-        "window_end": end_dt.isoformat(),
-        "query_log_count": len(entries),
-        "queries": entries,
-    }
-    with open(output_file, "w") as fh:
-        json.dump(manifest, fh, indent=2)
-    log.info("Query log manifest written to %s", output_file)
-
-    return manifest
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(
-        description="Collect BigQuery query logs into a JSON manifest",
-    )
-    parser.add_argument(
-        "--project-id",
-        default=os.getenv("BIGQUERY_PROJECT_ID"),
-        help="GCP project ID (or set BIGQUERY_PROJECT_ID env var)",
-    )
-    parser.add_argument("--lookback-hours", type=int, default=LOOKBACK_HOURS)
-    parser.add_argument("--lookback-lag-hours", type=int, default=LOOKBACK_LAG_HOURS)
-    parser.add_argument("--output-file", default="query_logs_output.json")
-    args = parser.parse_args()
-
-    if not args.project_id:
-        parser.error("--project-id or BIGQUERY_PROJECT_ID env var is required")
-
-    collect(
-        project_id=args.project_id,
-        lookback_hours=args.lookback_hours,
-        lookback_lag_hours=args.lookback_lag_hours,
-        output_file=args.output_file,
-    )
-
-
-if __name__ == "__main__":
-    main()

package/bundled-skills/monte-carlo-push-ingestion/scripts/templates/bigquery-iceberg/push_metadata.py

@@ -1,190 +0,0 @@
-"""
-BigQuery Iceberg — Metadata Push (push only)
-============================================
-Reads a JSON manifest produced by collect_metadata.py and pushes table
-metadata to Monte Carlo using the pycarlo SDK's IngestionService.
-
-Can be run standalone via CLI or imported (use the ``push()`` function).
-
-Substitution points (search for "← SUBSTITUTE"):
-  - MCD_INGEST_ID      : Monte Carlo Ingestion API key ID
-  - MCD_INGEST_TOKEN   : Monte Carlo Ingestion API key token
-  - MCD_RESOURCE_UUID  : Monte Carlo warehouse resource UUID
-
-Prerequisites:
-  pip install pycarlo>=0.12.251
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import logging
-import os
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from datetime import datetime, timezone
-
-from pycarlo.core import Client, Session
-from pycarlo.features.ingestion import IngestionService
-from pycarlo.features.ingestion.models import (
-    AssetField,
-    AssetFreshness,
-    AssetMetadata,
-    AssetVolume,
-    RelationalAsset,
-)
-
-logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
-log = logging.getLogger(__name__)
-
-RESOURCE_TYPE = "bigquery"
-_BATCH_SIZE = 500
-
-_ENDPOINT = "https://integrations.getmontecarlo.com"
-
-
-def _asset_from_dict(d: dict) -> RelationalAsset:
-    """Reconstruct a RelationalAsset from a manifest dict entry."""
-    fields = [
-        AssetField(
-            name=f["name"],
-            type=f.get("type"),
-            description=f.get("description"),
-        )
-        for f in d.get("fields", [])
-    ]
-
-    volume = None
-    if d.get("volume"):
-        volume = AssetVolume(
-            row_count=d["volume"].get("row_count"),
-            byte_count=d["volume"].get("byte_count"),
-        )
-
-    freshness = None
-    if d.get("freshness") and d["freshness"].get("last_updated_time"):
-        freshness = AssetFreshness(
-            last_update_time=d["freshness"]["last_updated_time"],
-        )
-
-    return RelationalAsset(
-        type=d.get("type", "TABLE"),
-        metadata=AssetMetadata(
-            name=d["name"],
-            database=d["database"],
-            schema=d["schema"],
-            description=d.get("description"),
-        ),
-        fields=fields,
-        volume=volume,
-        freshness=freshness,
-    )
-
-
-def push(
-    input_file: str,
-    resource_uuid: str,
-    key_id: str,
-    key_token: str,
-    batch_size: int = _BATCH_SIZE,
-    output_file: str = "metadata_push_result.json",
-) -> dict:
-    """Read a metadata manifest and push assets to Monte Carlo in batches."""
-    endpoint = _ENDPOINT
-    log.info("Using endpoint: %s", endpoint)
-    with open(input_file) as fh:
-        manifest = json.load(fh)
-
-    asset_dicts = manifest.get("assets", [])
-    resource_type = manifest.get("resource_type", RESOURCE_TYPE)
-    assets = [_asset_from_dict(d) for d in asset_dicts]
-    log.info("Loaded %d asset(s) from %s", len(assets), input_file)
-
-    batches = [assets[i : i + batch_size] for i in range(0, max(len(assets), 1), batch_size)]
-    total_batches = len(batches)
-
-    def _push_batch(batch: list[RelationalAsset], batch_num: int) -> str | None:
-        client = Client(session=Session(
-            mcd_id=key_id, mcd_token=key_token, scope="Ingestion", endpoint=endpoint,
-        ))
-        service = IngestionService(mc_client=client)
-        result = service.send_metadata(
-            resource_uuid=resource_uuid,
-            resource_type=resource_type,
-            events=batch,
-        )
-        invocation_id = service.extract_invocation_id(result)
-        log.info(
-            "Pushed batch %d/%d (%d assets) — invocation_id=%s",
-            batch_num, total_batches, len(batch), invocation_id,
-        )
-        return invocation_id
-
-    max_workers = min(4, total_batches)
-    invocation_ids: list[str | None] = [None] * total_batches
-
-    with ThreadPoolExecutor(max_workers=max_workers) as pool:
-        futures = {
-            pool.submit(_push_batch, batch, i + 1): i
-            for i, batch in enumerate(batches)
-        }
-        for future in as_completed(futures):
-            idx = futures[future]
-            try:
-                invocation_ids[idx] = future.result()
-            except Exception as exc:
-                log.error("ERROR pushing batch %d: %s", idx + 1, exc)
-                raise
-
-    log.info("All %d batch(es) pushed.", total_batches)
-
-    push_result = {
-        "resource_uuid": resource_uuid,
-        "resource_type": resource_type,
-        "invocation_ids": invocation_ids,
-        "pushed_at": datetime.now(timezone.utc).isoformat(),
-        "total_assets": len(assets),
-        "batch_count": total_batches,
-        "batch_size": batch_size,
-    }
-    with open(output_file, "w") as fh:
-        json.dump(push_result, fh, indent=2)
-    log.info("Push result written to %s", output_file)
-
-    return push_result
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(
-        description="Push BigQuery Iceberg metadata from a manifest to Monte Carlo",
-    )
-    parser.add_argument("--resource-uuid", default=os.getenv("MCD_RESOURCE_UUID"))
-    parser.add_argument("--key-id", default=os.getenv("MCD_INGEST_ID"))
-    parser.add_argument("--key-token", default=os.getenv("MCD_INGEST_TOKEN"))
-    parser.add_argument("--input-file", default="metadata_output.json")
-    parser.add_argument("--output-file", default="metadata_push_result.json")
-    parser.add_argument(
-        "--batch-size",
-        type=int,
-        default=_BATCH_SIZE,
-        help=f"Max assets per push batch (default: {_BATCH_SIZE})",
-    )
-    args = parser.parse_args()
-
-    required = ["resource_uuid", "key_id", "key_token"]
-    missing = [k for k in required if getattr(args, k) is None]
-    if missing:
-        parser.error(f"Missing required arguments/env vars: {missing}")
-
-    push(
-        input_file=args.input_file,
-        resource_uuid=args.resource_uuid,
-        key_id=args.key_id,
-        key_token=args.key_token,
-        batch_size=args.batch_size,
-        output_file=args.output_file,
-    )
-
-
-if __name__ == "__main__":
-    main()