opencode-skills-collection 2.0.0-beta.3 → 2.0.2

package/bundled-skills/monte-carlo-push-ingestion/scripts/templates/redshift/collect_and_push_query_logs.py

@@ -0,0 +1,88 @@
+"""
+Redshift — Query Log Collect & Push (combined)
+================================================
+Collects completed query execution records from Redshift using sys_query_history
+and sys_querytext, then pushes them to Monte Carlo for query-pattern analysis,
+lineage derivation, and usage attribution.
+
+This script imports and calls collect() from collect_query_logs and push() from
+push_query_logs, running both in sequence.
+
+Substitution points (search for "← SUBSTITUTE"):
+  - REDSHIFT_HOST / REDSHIFT_DB / REDSHIFT_USER / REDSHIFT_PASSWORD : connection
+  - LOOKBACK_HOURS    : hours back from [now - LAG_HOURS] to collect (default 25)
+  - LOOKBACK_LAG_HOURS: lag behind now to avoid in-flight queries (default 1)
+  - BATCH_SIZE        : number of query_ids to fetch texts for in one SQL call
+  - MAX_QUERIES       : maximum query rows to process per run
+  - MCD_INGEST_ID / MCD_INGEST_TOKEN : Monte Carlo API credentials
+  - MCD_RESOURCE_UUID  : UUID of the Redshift connection in Monte Carlo
+  - PUSH_BATCH_SIZE   : number of entries per API call (default 250)
+
+Prerequisites:
+  pip install psycopg2-binary pycarlo
+"""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import os
+
+from collect_query_logs import BATCH_SIZE, LOOKBACK_HOURS, LOOKBACK_LAG_HOURS, MAX_QUERIES, collect
+from push_query_logs import DEFAULT_BATCH_SIZE, push
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+log = logging.getLogger(__name__)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Collect and push Redshift query logs to Monte Carlo")
+    parser.add_argument("--host", default=os.getenv("REDSHIFT_HOST"))         # ← SUBSTITUTE
+    parser.add_argument("--db", default=os.getenv("REDSHIFT_DB"))             # ← SUBSTITUTE
+    parser.add_argument("--user", default=os.getenv("REDSHIFT_USER"))         # ← SUBSTITUTE
+    parser.add_argument("--password", default=os.getenv("REDSHIFT_PASSWORD")) # ← SUBSTITUTE
+    parser.add_argument("--port", type=int, default=int(os.getenv("REDSHIFT_PORT", "5439")))
+    parser.add_argument("--resource-uuid", default=os.getenv("MCD_RESOURCE_UUID"))
+    parser.add_argument("--key-id", default=os.getenv("MCD_INGEST_ID"))
+    parser.add_argument("--key-token", default=os.getenv("MCD_INGEST_TOKEN"))
+    parser.add_argument("--lookback-hours", type=int, default=LOOKBACK_HOURS)
+    parser.add_argument("--lookback-lag-hours", type=int, default=LOOKBACK_LAG_HOURS)
+    parser.add_argument("--batch-size", type=int, default=BATCH_SIZE)
+    parser.add_argument("--max-queries", type=int, default=MAX_QUERIES)
+    parser.add_argument("--push-batch-size", type=int, default=DEFAULT_BATCH_SIZE)
+    parser.add_argument("--manifest", default="manifest_query_logs.json")
+    args = parser.parse_args()
+
+    required = ["host", "db", "user", "password", "resource_uuid", "key_id", "key_token"]
+    missing = [k for k in required if getattr(args, k) is None]
+    if missing:
+        parser.error(f"Missing required arguments/env vars: {missing}")
+
+    log.info("Step 1: Collecting query logs …")
+    collect(
+        host=args.host,
+        db=args.db,
+        user=args.user,
+        password=args.password,
+        manifest_path=args.manifest,
+        port=args.port,
+        lookback_hours=args.lookback_hours,
+        lookback_lag_hours=args.lookback_lag_hours,
+        batch_size=args.batch_size,
+        max_queries=args.max_queries,
+    )
+
+    log.info("Step 2: Pushing query logs to Monte Carlo …")
+    push(
+        manifest_path=args.manifest,
+        resource_uuid=args.resource_uuid,
+        key_id=args.key_id,
+        key_token=args.key_token,
+        batch_size=args.push_batch_size,
+    )
+
+    log.info("Done — collect and push complete.")
+
+
+if __name__ == "__main__":
+    main()

package/bundled-skills/monte-carlo-push-ingestion/scripts/templates/redshift/collect_lineage.py

@@ -0,0 +1,235 @@
+"""
+Redshift — Lineage Collection (collect-only)
+==============================================
+Collects table-level lineage from Redshift by fetching recent successful query
+history from sys_query_history + sys_querytext and parsing CREATE TABLE AS SELECT
+(CTAS) and INSERT INTO SELECT patterns to derive source->destination relationships.
+
+Writes a JSON manifest file that can be consumed by push_lineage.py.
+
+Substitution points (search for "← SUBSTITUTE"):
+  - REDSHIFT_HOST / REDSHIFT_DB / REDSHIFT_USER / REDSHIFT_PASSWORD : connection
+  - LOOKBACK_HOURS    : how far back to scan query history (default 24 h)
+
+Prerequisites:
+  pip install psycopg2-binary
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import os
+import re
+from datetime import datetime, timezone
+from typing import Any
+
+import psycopg2
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+log = logging.getLogger(__name__)
+
+RESOURCE_TYPE = "redshift"
+LOOKBACK_HOURS: int = int(os.getenv("LOOKBACK_HOURS", "24"))  # ← SUBSTITUTE
+
+
+def _check_available_memory(min_gb: float = 2.0) -> None:
+    """Warn if available memory is below the threshold."""
+    try:
+        if hasattr(os, "sysconf"):  # Linux / macOS
+            page_size = os.sysconf("SC_PAGE_SIZE")
+            avail_pages = os.sysconf("SC_AVPHYS_PAGES")
+            avail_gb = (page_size * avail_pages) / (1024 ** 3)
+        else:
+            return  # Windows — skip check
+    except (ValueError, OSError):
+        return
+    if avail_gb < min_gb:
+        log.warning(
+            "Only %.1f GB of memory available (minimum recommended: %.1f GB). "
+            "Consider reducing the collection scope or increasing available memory.",
+            avail_gb,
+            min_gb,
+        )
+
+
+# Regex: CTAS — CREATE [OR REPLACE] TABLE <dest> AS SELECT
+_CTAS_RE = re.compile(
+    r"CREATE\s+(?:OR\s+REPLACE\s+)?(?:TABLE|VIEW)\s+(?P<dest>\"?[\w.\"]+\"?)\s*(?:\([^)]*\))?\s*AS\s+SELECT\b",
+    re.IGNORECASE | re.DOTALL,
+)
+# Regex: INSERT INTO <dest> … SELECT
+_INSERT_RE = re.compile(
+    r"INSERT\s+INTO\s+(?P<dest>\"?[\w.\"]+\"?)\s.*?SELECT\b",
+    re.IGNORECASE | re.DOTALL,
+)
+# Matches any schema.table or database.schema.table reference in the query
+_TABLE_REF_RE = re.compile(r'"?([\w]+)"?\."?([\w]+)"?(?:\."?([\w]+)"?)?', re.IGNORECASE)
+
+
+def _clean_name(name: str) -> str:
+    return name.strip('"').strip()
+
+
+def _parse_ref(ref: str) -> tuple[str, str, str]:
+    """Parse 'db.schema.table' or 'schema.table' -> (database, schema, table)."""
+    parts = [_clean_name(p) for p in ref.split(".")]
+    if len(parts) == 3:
+        return parts[0], parts[1], parts[2]
+    if len(parts) == 2:
+        return "", parts[0], parts[1]
+    return "", "", parts[0]
+
+
+def _dictfetch(cursor: Any, sql: str, params: tuple | None = None) -> list[dict[str, Any]]:
+    cursor.execute(sql, params)
+    cols = [d.name for d in cursor.description]
+    rows = []
+    while True:
+        chunk = cursor.fetchmany(1000)
+        if not chunk:
+            break
+        rows.extend(dict(zip(cols, row)) for row in chunk)
+    return rows
+
+
+def fetch_query_texts(cursor: Any, lookback_hours: int) -> list[str]:
+    """Assemble full query texts from sys_query_history + sys_querytext."""
+    rows = _dictfetch(
+        cursor,
+        f"""
+        SELECT
+            sq.query_id,
+            LISTAGG(
+                CASE WHEN LEN(st.text) <= 200 THEN st.text ELSE LEFT(st.text, 200) END,
+                ''
+            ) WITHIN GROUP (ORDER BY st.sequence) AS full_text
+        FROM sys_query_history sq
+        JOIN sys_querytext st ON sq.query_id = st.query_id
+        WHERE sq.start_time >= DATEADD(hour, -{lookback_hours}, GETDATE())
+          AND sq.status = 'success'
+        GROUP BY sq.query_id
+        LIMIT 50000
+        """,  # ← SUBSTITUTE: adjust lookback_hours, LIMIT, or add user/database filters
+    )
+    return [r["full_text"] for r in rows if r.get("full_text")]
+
+
+def parse_lineage_from_sql(sql_text: str) -> list[dict[str, Any]]:
+    events: list[dict[str, Any]] = []
+
+    dest_match = _CTAS_RE.search(sql_text) or _INSERT_RE.search(sql_text)
+    if not dest_match:
+        return events
+
+    dest_raw = dest_match.group("dest")
+    dest_db, dest_schema, dest_table = _parse_ref(dest_raw)
+    if not dest_table:
+        return events
+
+    # Find all schema.table refs in the query, excluding the destination
+    source_refs: list[str] = []
+    for m in _TABLE_REF_RE.finditer(sql_text):
+        if m.group(3):
+            ref = f"{m.group(1)}.{m.group(2)}.{m.group(3)}"
+        else:
+            ref = f"{m.group(1)}.{m.group(2)}"
+
+        db, schema, table = _parse_ref(ref)
+        if not table or (db == dest_db and schema == dest_schema and table == dest_table):
+            continue
+        source_refs.append(ref)
+
+    if not source_refs:
+        return events
+
+    # Deduplicate sources while preserving order
+    seen: set[str] = set()
+    sources: list[dict[str, str]] = []
+    for ref in source_refs:
+        if ref not in seen:
+            seen.add(ref)
+            db, schema, table = _parse_ref(ref)
+            sources.append({"database": db, "schema": schema, "asset_name": table})
+
+    events.append({
+        "sources": sources,
+        "destination": {"database": dest_db, "schema": dest_schema, "asset_name": dest_table},
+    })
+    return events
+
+
+def collect(
+    host: str,
+    db: str,
+    user: str,
+    password: str,
+    manifest_path: str = "manifest_lineage.json",
+    port: int = 5439,
+    lookback_hours: int = LOOKBACK_HOURS,
+) -> list[dict[str, Any]]:
+    """Connect to Redshift, collect lineage, write a JSON manifest, and return events."""
+    _check_available_memory()
+    collected_at = datetime.now(timezone.utc).isoformat()
+
+    conn = psycopg2.connect(
+        host=host, port=port, dbname=db, user=user, password=password, connect_timeout=30,
+    )
+    try:
+        with conn.cursor() as cursor:
+            query_texts = fetch_query_texts(cursor, lookback_hours)
+    finally:
+        conn.close()
+
+    log.info("Parsing lineage from %d query texts …", len(query_texts))
+    all_events: list[dict[str, Any]] = []
+    for sql_text in query_texts:
+        all_events.extend(parse_lineage_from_sql(sql_text))
+
+    log.info("Collected %d lineage events", len(all_events))
+
+    manifest = {
+        "resource_type": RESOURCE_TYPE,
+        "collected_at": collected_at,
+        "lookback_hours": lookback_hours,
+        "queries_scanned": len(query_texts),
+        "lineage_event_count": len(all_events),
+        "events": all_events,
+    }
+    with open(manifest_path, "w") as fh:
+        json.dump(manifest, fh, indent=2)
+    log.info("Manifest written to %s (%d events)", manifest_path, len(all_events))
+
+    return all_events
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Collect Redshift lineage to a manifest file")
+    parser.add_argument("--host", default=os.getenv("REDSHIFT_HOST"))         # ← SUBSTITUTE
+    parser.add_argument("--db", default=os.getenv("REDSHIFT_DB"))             # ← SUBSTITUTE
+    parser.add_argument("--user", default=os.getenv("REDSHIFT_USER"))         # ← SUBSTITUTE
+    parser.add_argument("--password", default=os.getenv("REDSHIFT_PASSWORD")) # ← SUBSTITUTE
+    parser.add_argument("--port", type=int, default=int(os.getenv("REDSHIFT_PORT", "5439")))
+    parser.add_argument("--lookback-hours", type=int, default=LOOKBACK_HOURS)
+    parser.add_argument("--manifest", default="manifest_lineage.json")
+    args = parser.parse_args()
+
+    required = ["host", "db", "user", "password"]
+    missing = [k for k in required if getattr(args, k) is None]
+    if missing:
+        parser.error(f"Missing required arguments/env vars: {missing}")
+
+    collect(
+        host=args.host,
+        db=args.db,
+        user=args.user,
+        password=args.password,
+        manifest_path=args.manifest,
+        port=args.port,
+        lookback_hours=args.lookback_hours,
+    )
+
+
+if __name__ == "__main__":
+    main()

package/bundled-skills/monte-carlo-push-ingestion/scripts/templates/redshift/collect_metadata.py

@@ -0,0 +1,219 @@
+"""
+Redshift — Metadata Collection (collect-only)
+===============================================
+Collects table schemas, row counts, and byte sizes from Amazon Redshift using
+SVV system views, then writes a JSON manifest file that can be consumed by
+push_metadata.py.
+
+Substitution points (search for "← SUBSTITUTE"):
+  - REDSHIFT_HOST     : Redshift cluster endpoint or serverless workgroup endpoint
+  - REDSHIFT_DB       : database name to connect to
+  - REDSHIFT_USER     : database user (or IAM role user)
+  - REDSHIFT_PASSWORD : database password
+  - DB_EXCLUSIONS     : databases to skip
+  - SCHEMA_EXCLUSIONS : schemas to skip in every database
+
+Prerequisites:
+  pip install psycopg2-binary
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import os
+from datetime import datetime, timezone
+from typing import Any
+
+import psycopg2
+import psycopg2.extras
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+log = logging.getLogger(__name__)
+
+RESOURCE_TYPE = "redshift"
+
+DB_EXCLUSIONS: set[str] = {"dev", "padb_harvest"}  # ← SUBSTITUTE: add internal databases
+
+SCHEMA_EXCLUSIONS: set[str] = {  # ← SUBSTITUTE: add internal schemas
+    "information_schema",
+    "pg_catalog",
+    "pg_internal",
+    "catalog_history",
+}
+
+
+def _check_available_memory(min_gb: float = 2.0) -> None:
+    """Warn if available memory is below the threshold."""
+    try:
+        if hasattr(os, "sysconf"):  # Linux / macOS
+            page_size = os.sysconf("SC_PAGE_SIZE")
+            avail_pages = os.sysconf("SC_AVPHYS_PAGES")
+            avail_gb = (page_size * avail_pages) / (1024 ** 3)
+        else:
+            return  # Windows — skip check
+    except (ValueError, OSError):
+        return
+    if avail_gb < min_gb:
+        log.warning(
+            "Only %.1f GB of memory available (minimum recommended: %.1f GB). "
+            "Consider reducing the collection scope or increasing available memory.",
+            avail_gb,
+            min_gb,
+        )
+
+
+def _dictfetch(cursor: Any, sql: str, params: tuple | None = None) -> list[dict[str, Any]]:
+    cursor.execute(sql, params)
+    cols = [d.name for d in cursor.description]
+    rows = []
+    while True:
+        chunk = cursor.fetchmany(1000)
+        if not chunk:
+            break
+        rows.extend(dict(zip(cols, row)) for row in chunk)
+    return rows
+
+
+def collect_databases(cursor: Any) -> list[str]:
+    rows = _dictfetch(
+        cursor,
+        "SELECT database_name FROM svv_redshift_databases ORDER BY database_name",
+    )
+    return [r["database_name"] for r in rows if r["database_name"] not in DB_EXCLUSIONS]
+
+
+def collect_tables(cursor: Any, db: str) -> list[dict[str, Any]]:
+    schema_list = ", ".join(f"'{s}'" for s in SCHEMA_EXCLUSIONS)
+    return _dictfetch(
+        cursor,
+        f"""
+        SELECT
+            database      AS db,
+            schema,
+            "table"       AS table_name,
+            "rows"        AS row_count,
+            size * 1024 * 1024 AS byte_count
+        FROM svv_table_info
+        WHERE database = %s
+          AND schema NOT IN ({schema_list})
+        ORDER BY schema, "table"
+        """,  # ← SUBSTITUTE: add additional WHERE clauses to narrow scope
+        (db,),
+    )
+
+
+def collect_columns(cursor: Any, db: str, schema: str, table: str) -> list[dict[str, Any]]:
+    return _dictfetch(
+        cursor,
+        """
+        SELECT column_name, data_type, remarks AS comment
+        FROM svv_columns
+        WHERE table_catalog = %s
+          AND table_schema  = %s
+          AND table_name    = %s
+        ORDER BY ordinal_position
+        """,
+        (db, schema, table),
+    )
+
+
+def collect(
+    host: str,
+    db: str,
+    user: str,
+    password: str,
+    manifest_path: str = "manifest_metadata.json",
+    port: int = 5439,
+) -> list[dict[str, Any]]:
+    """Connect to Redshift, collect metadata, write a JSON manifest, and return asset dicts."""
+    _check_available_memory()
+    collected_at = datetime.now(timezone.utc).isoformat()
+    assets: list[dict[str, Any]] = []
+
+    conn = psycopg2.connect(
+        host=host,          # ← SUBSTITUTE
+        port=port,
+        dbname=db,          # ← SUBSTITUTE
+        user=user,          # ← SUBSTITUTE
+        password=password,  # ← SUBSTITUTE
+        connect_timeout=30,
+    )
+    try:
+        with conn.cursor() as cursor:
+            databases = collect_databases(cursor)
+            log.info("Found databases: %s", databases)
+
+            for database in databases:
+                tables = collect_tables(cursor, database)
+                log.info("Database %s — %d tables", database, len(tables))
+
+                for t in tables:
+                    schema = t["schema"]
+                    table_name = t["table_name"]
+
+                    columns = collect_columns(cursor, database, schema, table_name)
+                    fields = [
+                        {
+                            "name": col["column_name"],
+                            "type": col["data_type"].upper(),
+                            "description": col.get("comment") or None,
+                        }
+                        for col in columns
+                    ]
+
+                    asset = {
+                        "asset_name": table_name,
+                        "database": database,   # ← SUBSTITUTE: use database as top-level namespace
+                        "schema": schema,
+                        "asset_type": "TABLE",
+                        "fields": fields,
+                        "row_count": t.get("row_count"),
+                        "byte_count": t.get("byte_count"),
+                    }
+                    assets.append(asset)
+                    log.info("Collected %s.%s.%s", database, schema, table_name)
+    finally:
+        conn.close()
+
+    manifest = {
+        "resource_type": RESOURCE_TYPE,
+        "collected_at": collected_at,
+        "asset_count": len(assets),
+        "assets": assets,
+    }
+    with open(manifest_path, "w") as fh:
+        json.dump(manifest, fh, indent=2)
+    log.info("Manifest written to %s (%d assets)", manifest_path, len(assets))
+
+    return assets
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Collect Redshift metadata to a manifest file")
+    parser.add_argument("--host", default=os.getenv("REDSHIFT_HOST"))         # ← SUBSTITUTE
+    parser.add_argument("--db", default=os.getenv("REDSHIFT_DB"))             # ← SUBSTITUTE
+    parser.add_argument("--user", default=os.getenv("REDSHIFT_USER"))         # ← SUBSTITUTE
+    parser.add_argument("--password", default=os.getenv("REDSHIFT_PASSWORD")) # ← SUBSTITUTE
+    parser.add_argument("--port", type=int, default=int(os.getenv("REDSHIFT_PORT", "5439")))
+    parser.add_argument("--manifest", default="manifest_metadata.json")
+    args = parser.parse_args()
+
+    required = ["host", "db", "user", "password"]
+    missing = [k for k in required if getattr(args, k) is None]
+    if missing:
+        parser.error(f"Missing required arguments/env vars: {missing}")
+
+    collect(
+        host=args.host,
+        db=args.db,
+        user=args.user,
+        password=args.password,
+        manifest_path=args.manifest,
+        port=args.port,
+    )
+
+
+if __name__ == "__main__":
+    main()