@hraza01/skyhook 1.0.0

package/README.md

@@ -0,0 +1,75 @@
+# Skyhook 🪂
+
+**Skyhook** is a robust, interactive CLI tool for deploying Airflow DAGs to Google Cloud Composer. It replaces legacy bash scripts with a modern Node.js interface featuring Git validation, synchronized spinners, and a professional UI.
+
+## About
+
+> **Use Case**: Skyhook is optimized for development environments and teams where a full CI/CD pipeline is not yet established. While automated CI/CD remains the industry standard for production deployments, this utility provides a safe, structured, and efficient alternative for manual deployments, helping smaller teams iterate faster without sacrificing validation or safety.
+
+## Features
+
+- **🛡️ Safety First**: Automatically validates that your DAG folder is a Git repository, is on the `main` branch, and is fully synced with remote.
+- **✨ Modern UI**: Built with `@clack/prompts` for a minimal, clean aesthetic.
+- **🚀 Live Feedback**: Shows real-time file upload progress from `gsutil`.
+- **🔌 Cross-Platform**: Works seamlessly on macOS, Linux, and Windows (WSL).
+
+## Prerequisites
+
+- **Node.js**: v20.5.0 or higher
+- **Google Cloud SDK**: `gsutil` must be installed and **authenticated**.
+- **Git**: Must be installed and available in the PATH.
+- **Environment**: You must be inside your Composer's local Airflow development environment (project root) to run this tool.
+
+## Installation
+
+Install globally via NPM:
+
+```console
+$ npm install -g @hraza01/skyhook
+```
+
+## Updating
+
+To update to the latest version:
+
+```console
+$ npm update -g @hraza01/skyhook
+```
+
+## Configuration
+
+Skyhook requires two environment variables to function. It will **exit** if these are not provided.
+
+| Variable            | Description                                    |
+| :------------------ | :--------------------------------------------- |
+| `GCS_BUCKET_NAME`   | The name of your Composer GCS bucket.          |
+| `COMPOSER_URL_BASE` | The base URL for your environment's webserver. |
+
+## Usage
+
+### Option 1: VS Code Task (Recommended)
+
+1.  Copy the example task configuration from `vscode-example/tasks.json` to your project's `.vscode/tasks.json`.
+2.  Update the `env` variables in `tasks.json`.
+3.  Run the task **"Deploy DAG to Cloud Composer"**.
+
+### Option 2: CLI (Manual)
+
+Navigate to your Airflow project root (where the `dags/` folder is) and run:
+
+```console
+$ export GCS_BUCKET_NAME="<your-bucket-name>"
+$ export COMPOSER_URL_BASE="<your-composer-webserver-url-base>"
+
+$ skyhook
+```
+
+## Project Structure
+
+For a detailed overview of the project structure and module descriptions, see:
+
+📄 **[Project Structure Documentation](https://github.com/hraza01/skyhook/blob/main/docs/project-structure.md)**
+
+## Thank You
+
+Thank you for your attention and for taking the time to read the documentation! We hope this tool helps streamline your workflow.

package/package.json

@@ -0,0 +1,39 @@
+{
+    "name": "@hraza01/skyhook",
+    "version": "1.0.0",
+    "description": "Interactive CLI for Cloud Composer DAG deployment",
+    "main": "src/index.js",
+    "bin": {
+        "skyhook": "src/index.js"
+    },
+    "scripts": {
+        "test": "echo \"Error: no test specified\" && exit 1"
+    },
+    "keywords": [
+        "airflow",
+        "cloud-composer",
+        "dag",
+        "deployment",
+        "cli",
+        "gcp"
+    ],
+    "author": "Hasan Raza",
+    "license": "MIT",
+    "type": "module",
+    "repository": {
+        "type": "git",
+        "url": "https://github.com/hraza01/skyhook.git"
+    },
+    "dependencies": {
+        "@clack/prompts": "^1.0.0",
+        "chalk": "^5.6.2",
+        "execa": "^9.6.1",
+        "terminal-link": "^5.0.0"
+    },
+    "engines": {
+        "node": ">=20.5.0"
+    },
+    "devDependencies": {
+        "prettier": "^3.8.1"
+    }
+}

package/shell/deploy_core.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+set -e
+
+# Configuration
+if [ -z "$GCS_BUCKET_NAME" ]; then
+    echo "Error: GCS_BUCKET_NAME environment variable is not set."
+    exit 1
+fi
+
+# Input arguments
+SOURCE_PATH="$1"
+DEST_PATH="$2"
+
+if [ -z "$SOURCE_PATH" ] || [ -z "$DEST_PATH" ]; then
+    echo "Error: Missing arguments. Usage: $0 <SOURCE_PATH> <DEST_PATH>"
+    exit 1
+fi
+
+# Validation is handled by the calling Node.js script. We assume valid inputs here.
+echo "Syncing $SOURCE_PATH to $DEST_PATH..."
+
+# Calculate file count (informational)
+FILE_COUNT=$(find "$SOURCE_PATH" -type f -not -path '*/.git/*' -not -path '*/__pycache__/*' | wc -l | tr -d ' ')
+echo "Uploading $FILE_COUNT files..."
+
+# Perform Sync
+gsutil -m rsync -r -x "\.git/.*|__pycache__/.*" "$SOURCE_PATH" "$DEST_PATH"

package/src/cli.js

@@ -0,0 +1,53 @@
+import chalk from "chalk"
+import { intro, spinner } from "@clack/prompts"
+
+/**
+ * Display version information
+ */
+export function showVersionInfo() {
+    console.log("skyhook v1.0.0")
+    process.exit(0)
+}
+
+/**
+ * Display help information
+ */
+export function showHelpInfo() {
+    console.log(
+        chalk.cyan.bold("\nSkyhook - Cloud Composer DAG Deployment Utility\n"),
+    )
+    console.log("Usage:")
+    console.log("  skyhook [options] [path]\n")
+    console.log("Options:")
+    console.log("  -h, --help     Show this help message")
+    console.log("  --version      Show version number")
+    console.log("  -v, --verbose  Enable verbose logging\n")
+    console.log("Environment Variables (Required):")
+    console.log("  GCS_BUCKET_NAME    Your Composer GCS bucket name")
+    console.log("  COMPOSER_URL_BASE  Your Composer webserver base URL\n")
+    console.log("For more information, visit:")
+    console.log("  https://github.com/hraza01/skyhook\n")
+    process.exit(0)
+}
+
+/**
+ * Display the intro banner
+ */
+export function showIntro() {
+    intro(
+        chalk.bgCyan(
+            chalk.black("  Skyhook / Cloud Composer Deployment Utility  "),
+        ),
+    )
+}
+
+/**
+ * Create and return a configured spinner with cyan color
+ */
+export function createSpinner() {
+    return spinner({
+        frames: ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"].map(
+            (frame) => chalk.cyan(frame),
+        ),
+    })
+}

package/src/config.js

@@ -0,0 +1,42 @@
+import { parseArgs } from "util"
+import path from "path"
+import { ConfigError } from "./errors.js"
+
+const { values, positionals } = parseArgs({
+    options: {
+        verbose: {
+            type: "boolean",
+            short: "v",
+        },
+        version: {
+            type: "boolean",
+        },
+        help: {
+            type: "boolean",
+            short: "h",
+        },
+    },
+    allowPositionals: true,
+})
+
+export const verbose = values.verbose
+export const showVersion = values.version
+export const showHelp = values.help
+
+// Helper to get root dir based on arg or CWD
+export const ROOT_DIR = positionals[0]
+    ? path.resolve(positionals[0])
+    : process.cwd()
+export const DAGS_DIR = path.join(ROOT_DIR, "dags")
+
+export function validateEnv() {
+    if (!process.env.GCS_BUCKET_NAME || !process.env.COMPOSER_URL_BASE) {
+        throw new ConfigError(
+            "Missing GCS_BUCKET_NAME or COMPOSER_URL_BASE environment variables.",
+        )
+    }
+}
+
+export const GCS_BUCKET_NAME = process.env.GCS_BUCKET_NAME
+export const COMPOSER_URL_BASE = process.env.COMPOSER_URL_BASE
+export const BUCKET_URL = `gs://${GCS_BUCKET_NAME}/dags`

package/src/dag_selection.js

@@ -0,0 +1,46 @@
+import fs from "fs"
+import path from "path"
+import { select, isCancel } from "@clack/prompts"
+import { logger } from "./logger.js"
+import { UserCancellationError } from "./errors.js"
+
+export function scanDags(dagsDir, s) {
+    logger.info("SCAN", `Scanning directory: ${dagsDir}`)
+    s.start(`Looking for DAGs in: ${path.relative(process.cwd(), dagsDir)}`)
+
+    if (!fs.existsSync(dagsDir)) {
+        s.stop(`Directory '${dagsDir}' not found.`, 1)
+        throw new UserCancellationError("Operation cancelled.")
+    }
+
+    const folders = fs.readdirSync(dagsDir).filter((file) => {
+        return (
+            fs.statSync(path.join(dagsDir, file)).isDirectory() &&
+            !file.startsWith(".")
+        )
+    })
+
+    if (folders.length === 0) {
+        s.stop(`No folders found in ${dagsDir}`, 1)
+        throw new UserCancellationError("Operation cancelled.")
+    }
+
+    s.stop(`Found ${folders.length} Airflow DAGs.`)
+    logger.info("SCAN", `Found ${folders.length} valid DAG folders.`)
+    return folders
+}
+
+export async function selectDag(folders) {
+    const selectedFolder = await select({
+        message: "Select an Airflow DAG to deploy:",
+        options: folders.map((f) => ({ label: f, value: f })),
+    })
+
+    if (isCancel(selectedFolder)) {
+        logger.warn("SELECT", "User cancelled selection.")
+        throw new UserCancellationError("Deployment cancelled.")
+    }
+
+    logger.info("SELECT", `User selected: ${selectedFolder}`)
+    return selectedFolder
+}

package/src/deploy.js

@@ -0,0 +1,153 @@
+import { execa } from "execa"
+import chalk from "chalk"
+import path from "path"
+import { cancel, outro } from "@clack/prompts"
+import terminalLink from "terminal-link"
+import { fileURLToPath } from "url"
+import { countFiles } from "./utils.js"
+import { logger } from "./logger.js"
+import { BUCKET_URL, COMPOSER_URL_BASE } from "./config.js"
+
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+
+export async function deployDag(selectedFolder, sourcePath, s, verbose) {
+    s.start("Starting GCS Upload...")
+    logger.info(
+        "DEPLOY",
+        `Starting deployment for ${selectedFolder} to ${BUCKET_URL}`,
+    )
+
+    try {
+        const deployScriptPath = path.resolve(
+            __dirname,
+            "../shell/deploy_core.sh",
+        )
+        const destination = `${BUCKET_URL}/${selectedFolder}`
+        const subprocess = execa(
+            "bash",
+            [deployScriptPath, sourcePath, destination],
+            {
+                all: true,
+            },
+        )
+
+        const ignorePatterns = [
+            "WARNING: gsutil rsync uses hashes",
+            "module's C extension",
+            "checksumming will run very slowly",
+            "gsutil help crcmod",
+            "Building synchronization state...",
+            "problems with multiprocessing",
+            "python.org/issue33725",
+            "parallel_process_count",
+            "multithreading is still available",
+        ]
+
+        subprocess.all.on("data", (chunk) => {
+            const lines = chunk.toString().split("\n")
+            for (const line of lines) {
+                if (!line.trim()) continue
+
+                const shouldIgnore = ignorePatterns.some((pattern) =>
+                    line.includes(pattern),
+                )
+                if (shouldIgnore) continue
+
+                if (line.includes("Uploading") && line.includes("files...")) {
+                    s.message(line.trim())
+                } else if (line.includes("Copying")) {
+                    // Output looks like: "Copying file://.../dags/my_folder/sub/file.py [Content-Type=...]"
+                    // We want to extract: "my_folder/sub/file.py"
+
+                    // 1. Remove metadata suffix (e.g. [Content-Type...])
+                    let cleanLine = line.split("[")[0].trim()
+
+                    // 2. Find the index where 'selectedFolder' starts
+                    const folderIndex = cleanLine.indexOf(selectedFolder)
+
+                    if (folderIndex !== -1) {
+                        // Extract substring starting from selectedFolder
+                        const relativePath = cleanLine.substring(folderIndex)
+                        s.message(`Syncing: ${relativePath}...`)
+                        logger.info("SYNC", `Syncing: ${relativePath}`)
+                    } else {
+                        // Fallback if folder name not found (unlikely but safe)
+                        const match = cleanLine.match(/([^/]+)$/) // Get filename
+                        if (match && match[1]) {
+                            s.message(`Syncing: ${match[1]}...`)
+                            logger.info("SYNC", `Syncing: ${match[1]}`)
+                        } else {
+                            s.message(`Syncing...`)
+                        }
+                    }
+                }
+            }
+        })
+
+        await subprocess
+        s.stop("GCS sync complete.")
+
+        // Show Summary
+        const fileCount = countFiles(sourcePath)
+
+        // Simple padding helper
+        const pad = (str, len) => str.padEnd(len)
+        const labelWidth = 15
+
+        const productionUrl = `${COMPOSER_URL_BASE}/dags/${selectedFolder}`
+        const link = terminalLink(
+            chalk.cyan.underline("Open in Cloud Console"),
+            productionUrl,
+        )
+
+        console.log("")
+        console.log(chalk.green("Deployment Summary"))
+        console.log("")
+
+        console.log(
+            `${chalk.dim(pad("Source", labelWidth))} ${chalk.reset(
+                `dags/${selectedFolder}`,
+            )}`,
+        )
+        console.log(
+            `${chalk.dim(pad("Destination", labelWidth))} ${chalk.reset(
+                `${BUCKET_URL}/${selectedFolder}`,
+            )}`,
+        )
+        console.log(
+            `${chalk.dim(pad("Files Synced", labelWidth))} ${chalk.reset(
+                `${fileCount} files`,
+            )}`,
+        )
+
+        console.log("")
+        console.log(
+            chalk.white(
+                `${chalk.bold(
+                    selectedFolder,
+                )} is now in sync with git + Cloud Composer.`,
+            ),
+        )
+        console.log("")
+
+        // Print URL in original location but single line
+        console.log(chalk.dim(pad("Composer URL", labelWidth)) + link)
+        console.log("")
+
+        outro(chalk.green.bold("Deployment Successful!"))
+        logger.info("DEPLOY", "Deployment steps completed successfully.")
+    } catch (error) {
+        logger.error("DEPLOY", `Deployment failed: ${error.message}`)
+        s.stop("Deployment Failed ❌", 1)
+        console.log(chalk.red("\nError Logs:"))
+        if (error.all) {
+            console.log(error.all)
+        } else {
+            console.log(error.message)
+            console.log(error)
+        }
+        cancel("Correct the errors above and try again.")
+        process.exit(1)
+    }
+}

package/src/errors.js

@@ -0,0 +1,20 @@
+export class UserCancellationError extends Error {
+    constructor(message = "Operation cancelled by user") {
+        super(message)
+        this.name = "UserCancellationError"
+    }
+}
+
+export class ValidationError extends Error {
+    constructor(message) {
+        super(message)
+        this.name = "ValidationError"
+    }
+}
+
+export class ConfigError extends Error {
+    constructor(message) {
+        super(message)
+        this.name = "ConfigError"
+    }
+}

package/src/git_validation.js

@@ -0,0 +1,75 @@
+import { execa } from "execa"
+import chalk from "chalk"
+import { ValidationError } from "./errors.js"
+
+export async function validateGit(sourcePath, s) {
+    s.start("Validating Git status...")
+
+    // A. Check if valid git repo
+    try {
+        await execa("git", [
+            "-C",
+            sourcePath,
+            "rev-parse",
+            "--is-inside-work-tree",
+        ])
+    } catch (e) {
+        s.stop("Validation Failed: Not a git repository.", 1)
+        throw new ValidationError("The DAG folder must be version controlled.")
+    }
+
+    // B. Check Branch Name
+    const { stdout: branch } = await execa("git", [
+        "-C",
+        sourcePath,
+        "rev-parse",
+        "--abbrev-ref",
+        "HEAD",
+    ])
+    if (branch.trim() !== "main") {
+        s.stop(`Validation Failed: You are on branch '${branch.trim()}'.`, 1)
+        throw new ValidationError("You must be on the 'main' branch to deploy.")
+    }
+
+    // C. Check Sync Status
+    s.message("Checking remote sync status...")
+    await execa("git", ["-C", sourcePath, "fetch", "origin", "main"])
+
+    const { stdout: localHash } = await execa("git", [
+        "-C",
+        sourcePath,
+        "rev-parse",
+        "main",
+    ])
+    const { stdout: remoteHash } = await execa("git", [
+        "-C",
+        sourcePath,
+        "rev-parse",
+        "origin/main",
+    ])
+
+    if (localHash.trim() !== remoteHash.trim()) {
+        s.stop("Validation Failed: Branch is out of sync with origin/main.", 1)
+
+        // Optional: Show ahead/behind info
+        try {
+            const { stdout: counts } = await execa("git", [
+                "-C",
+                sourcePath,
+                "rev-list",
+                "--left-right",
+                "--count",
+                "main...origin/main",
+            ])
+            const [ahead, behind] = counts.trim().split(/\s+/).map(Number)
+            if (ahead > 0)
+                console.log(chalk.yellow(`   - Ahead by ${ahead} commit(s)`))
+            if (behind > 0)
+                console.log(chalk.yellow(`   - Behind by ${behind} commit(s)`))
+        } catch (ignored) {}
+
+        throw new ValidationError("Please pull/push changes before deploying.")
+    }
+
+    s.stop("Git validation passed (main branch, in sync).")
+}

package/src/index.js

@@ -0,0 +1,86 @@
+#!/usr/bin/env node
+
+import chalk from "chalk"
+import { cancel } from "@clack/prompts"
+import {
+    validateEnv,
+    DAGS_DIR,
+    verbose,
+    showVersion,
+    showHelp,
+} from "./config.js"
+import {
+    showVersionInfo,
+    showHelpInfo,
+    showIntro,
+    createSpinner,
+} from "./cli.js"
+import { scanDags, selectDag } from "./dag_selection.js"
+import { validateGit } from "./git_validation.js"
+import { deployDag } from "./deploy.js"
+import { fetchQuote } from "./quotes.js"
+import { initLogger, logger } from "./logger.js"
+import path from "path"
+
+async function main() {
+    try {
+        console.clear()
+        initLogger(verbose)
+        logger.info("INIT", "Skyhook started")
+
+        // Handle --version flag
+        if (showVersion) showVersionInfo()
+
+        // Handle --help flag
+        if (showHelp) showHelpInfo()
+
+        validateEnv()
+        showIntro()
+
+        const s = createSpinner()
+
+        // 1. Scan
+        const folders = scanDags(DAGS_DIR, s)
+
+        // 2. Select
+        const selectedFolder = await selectDag(folders)
+        const sourcePath = path.join(DAGS_DIR, selectedFolder)
+
+        // 3. Validate
+        await validateGit(sourcePath, s)
+
+        // 4. Deploy
+        await deployDag(selectedFolder, sourcePath, s, verbose)
+
+        // 5. Post-Deployment Polish
+        const quote = await fetchQuote()
+        if (quote) {
+            console.log(chalk.italic.dim(`\n${quote}\n`))
+        }
+    } catch (error) {
+        if (error.name === "UserCancellationError") {
+            cancel(error.message)
+            process.exit(0)
+        } else if (error.name === "ValidationError") {
+            cancel(chalk.red(error.message))
+            process.exit(0)
+        } else if (error.name === "ConfigError") {
+            console.log(chalk.red("Configuration Error:"))
+            console.log(chalk.yellow(error.message))
+            process.exit(0)
+        } else {
+            throw error // Re-throw unexpected errors
+        }
+    }
+}
+
+main().catch((err) => {
+    console.error(chalk.red("\nUnexpected Error:"))
+    console.error(chalk.red(err.message || err))
+
+    if (verbose) {
+        console.error(chalk.dim(err.stack))
+    }
+
+    process.exit(1)
+})

package/src/logger.js

@@ -0,0 +1,36 @@
+import fs from "fs"
+import path from "path"
+import { ROOT_DIR } from "./config.js"
+
+const LOG_FILE = path.join(ROOT_DIR, "skyhook.log")
+
+let isVerbose = false
+
+export function initLogger(verbose = false) {
+    isVerbose = verbose
+    if (isVerbose) {
+        try {
+            fs.writeFileSync(LOG_FILE, "")
+        } catch (e) {
+            // Warning: unable to write logs
+        }
+    }
+}
+
+export function log(step, level, message) {
+    if (!isVerbose) return
+    const timestamp = new Date().toISOString()
+    const formattedMessage = `[${timestamp}][${step}][${level}] ${message}`
+
+    try {
+        fs.appendFileSync(LOG_FILE, formattedMessage + "\n")
+    } catch (e) {
+        // Fail silently if logging fails
+    }
+}
+
+export const logger = {
+    info: (step, message) => log(step, "INFO", message),
+    warn: (step, message) => log(step, "WARN", message),
+    error: (step, message) => log(step, "ERROR", message),
+}

package/src/quotes.js

@@ -0,0 +1,17 @@
+export async function fetchQuote() {
+    const categories = ["inspiration", "excellence", "truth", "success"]
+    const category = categories[Math.floor(Math.random() * categories.length)]
+
+    try {
+        const response = await fetch(
+            `https://zenquotes.io/api/random/${category}`,
+        )
+        const data = await response.json()
+        if (data && data[0] && data[0].q && data[0].a) {
+            return `"${data[0].q}" — ${data[0].a}`
+        }
+    } catch (e) {
+        // Fallback or silence
+    }
+    return null
+}

package/src/utils.js

@@ -0,0 +1,19 @@
+import fs from "fs"
+import path from "path"
+
+export function countFiles(dir) {
+    let results = 0
+    const list = fs.readdirSync(dir)
+    list.forEach((file) => {
+        file = path.resolve(dir, file)
+        const stat = fs.statSync(file)
+        if (stat && stat.isDirectory()) {
+            if (!file.includes(".git") && !file.includes("__pycache__")) {
+                results += countFiles(file)
+            }
+        } else {
+            results += 1
+        }
+    })
+    return results
+}