create-airframe-dag 1.0.0

package/README.md

@@ -0,0 +1,64 @@
+# Airframe
+
+> The blueprint for scalable Airflow DAGs
+
+Airframe is a CLI tool designed to scaffold production-ready Apache Airflow DAG projects with best practices built-in. It helps you quickly customize and generate DAG structures from curated templates, ensuring consistency and scalability across your data pipelines.
+
+## Features
+
+- 🚀 **Interactive Scaffolding**: Easy-to-use CLI wizard to set up your project.
+- 📦 **Curated Templates**: Choose from a variety of pre-defined DAG patterns (fetched dynamically from the [Airframe Template Registry](https://github.com/hraza01/airframe-templates)).
+- 🔧 **Best Practices**: Enforces a structured project layout within your `dags/` directory.
+- 🛠 **Automated Setup**: Handles directory creation, variable substitution, and Git initialization.
+
+## Prerequisites
+
+- **Node.js**: Version 20.5.0 or higher.
+- **Airflow Environment**: You must run this tool from the root of your Airflow project (specifically, a directory that contains a `dags/` folder).
+
+## Usage
+
+To start a new Airframe project, run the following command in your terminal:
+
+```bash
+npx create-airframe-dag@latest
+```
+
+Follow the interactive prompts to:
+
+1.  Define your project details (Name, Author).
+2.  Select a template.
+3.  Scaffold the project into your `dags/` directory.
+
+### Options
+
+- `--airflow-path <path>`: Specify the path to your Airflow root directory if you are not running the command from there.
+- `-v, --version`: Show the current CLI version.
+- `-h, --help`: Show help information.
+
+## How It Works
+
+1.  **Checks Environment**: Verifies that a `dags/` directory exists in the current (or specified) location.
+2.  **Fetches Templates**: Retrieves the latest templates from the central registry.
+3.  **Collects Details**: Prompts you for formatting configuration and project metadata.
+4.  **Generates Code**: Creates a new directory `dags/<project-name>`, generates the boilerplate code for the selected template, and customizes it with your provided details.
+5.  **Git Init**: Initializes a new Git repository within the specific project folder for version control.
+
+## Project Structure
+
+Airframe creates the following structure for each project:
+
+```
+your-airflow-env/
+└── dags/
+    └── <airframe_project_name>/   # Created by Airframe
+        ├── README.md              # Project-specific documentation
+        ├── .gitignore             # Git ignore file for the project
+        ├── main.py                # DAG definition
+        ├── sql/                   # Directory for SQL files
+        └── ...                    # Other files
+```
+
+## Contributing
+
+Created by [@hraza01](https://github.com/hraza01).

package/bin/cli.js

@@ -0,0 +1,9 @@
+#!/usr/bin/env node
+
+import { main } from "../src/main.js"
+import chalk from "chalk"
+
+main().catch((err) => {
+    console.error(chalk.red("Unexpected error:"), err)
+    process.exit(1)
+})

package/package.json

@@ -0,0 +1,45 @@
+{
+    "name": "create-airframe-dag",
+    "version": "1.0.0",
+    "description": "CLI tool to scaffold Airflow DAG projects",
+    "main": "bin/cli.js",
+    "type": "module",
+    "bin": {
+        "create-airframe-dag": "./bin/cli.js"
+    },
+    "scripts": {
+        "start": "node bin/cli.js",
+        "test": "echo \"Error: no test specified\" && exit 1"
+    },
+    "keywords": [
+        "cli",
+        "airflow",
+        "cloud-composer",
+        "scaffold",
+        "dag",
+        "gcp"
+    ],
+    "author": "Hasan Raza",
+    "license": "MIT",
+    "dependencies": {
+        "@clack/prompts": "^1.0.0",
+        "chalk": "^5.6.2",
+        "execa": "^9.6.1",
+        "figlet": "^1.10.0",
+        "fs-extra": "^11.3.3",
+        "ora": "^9.3.0",
+        "terminal-link": "^5.0.0",
+        "tiged": "^2.12.7"
+    },
+    "overrides": {
+        "tiged": {
+            "tar": "^7.5.7"
+        }
+    },
+    "engines": {
+        "node": ">=20.5.0"
+    },
+    "devDependencies": {
+        "prettier": "^3.8.1"
+    }
+}

package/src/api.js

@@ -0,0 +1,17 @@
+export async function fetchTemplates() {
+    const MANIFEST_URL =
+        "https://raw.githubusercontent.com/hraza01/airframe-templates/main/templates.json"
+
+    try {
+        const response = await fetch(MANIFEST_URL)
+        if (!response.ok) {
+            throw new Error(`Failed to fetch templates: ${response.statusText}`)
+        }
+        const data = await response.json()
+        return data
+    } catch (error) {
+        throw new Error(
+            `Could not connect to template registry: ${error.message}`,
+        )
+    }
+}

package/src/commands/init.js

@@ -0,0 +1,99 @@
+import { fetchTemplates } from "../api.js"
+import { scaffoldProject, customizeProject } from "../scaffolder.js"
+import {
+    checkDagsDirectory,
+    formatTargetDir,
+    getExistingProjects,
+} from "../lib/env.js"
+import { initGitRepo } from "../lib/git.js"
+import { showWelcome, displayTemplates, showCompletion } from "../ui/display.js"
+import {
+    createSpinner,
+    askForProjectDetails,
+    selectTemplate,
+} from "../ui/prompts.js"
+import { log } from "@clack/prompts"
+import chalk from "chalk"
+import readline from "node:readline"
+
+export async function initCommand({ airflowPath } = {}) {
+    const baseDir = airflowPath ? String(airflowPath) : process.cwd()
+
+    await showWelcome(baseDir)
+
+    // Step 0: Check for dags directory
+    if (!checkDagsDirectory(baseDir)) {
+        log.error(`No "dags" directory found in path: ${baseDir}`)
+        log.warn(
+            "Please run `airframe init` from your airflow environment directory.",
+        )
+        process.exit(1)
+    }
+
+    const s = createSpinner()
+
+    // Step 1: Fetch Templates
+    let templates
+    try {
+        templates = await fetchTemplates()
+    } catch (error) {
+        console.error(chalk.red(error.message))
+        process.exit(1)
+    }
+
+    // Step 2: Project Details
+    const existingProjects = await getExistingProjects(baseDir)
+    const { projectName, dagAuthor } =
+        await askForProjectDetails(existingProjects)
+
+    // Step 3: Select Template
+    displayTemplates(templates)
+    const template = await selectTemplate(templates)
+
+    const targetDir = formatTargetDir(projectName, baseDir)
+
+    // Step 4: Scaffold
+    console.log(chalk.dim("│"))
+    s.start(`Scaffolding project in ${targetDir}...`)
+    try {
+        await scaffoldProject({ template, targetDir })
+        s.stop()
+        readline.clearLine(process.stdout, 0)
+        readline.cursorTo(process.stdout, 0)
+        log.success("Scaffolding complete.")
+    } catch (error) {
+        s.stop()
+        readline.clearLine(process.stdout, 0)
+        readline.cursorTo(process.stdout, 0)
+        log.error("Scaffolding failed.")
+        console.error(chalk.red(error.message))
+        process.exit(1)
+    }
+
+    // Step 5: Setup project
+    s.start("Setting up project...")
+    try {
+        await customizeProject({ targetDir, projectName, dagAuthor, template })
+        s.stop()
+        readline.clearLine(process.stdout, 0)
+        readline.cursorTo(process.stdout, 0)
+        log.success("Project setup complete.")
+    } catch (error) {
+        s.stop()
+        readline.clearLine(process.stdout, 0)
+        readline.cursorTo(process.stdout, 0)
+        log.error("Project setup failed.")
+        console.error(chalk.red(error.message))
+        process.exit(1)
+    }
+
+    // Step 6: Git Init
+    try {
+        await initGitRepo(targetDir)
+    } catch (error) {
+        log.warn(`Failed to initialize git repository: ${error.message}`)
+    }
+
+    // Step 7: Done
+    showCompletion({ targetDir })
+}

package/src/lib/env.js

@@ -0,0 +1,20 @@
+import fs from "fs-extra"
+import path from "path"
+
+export function formatTargetDir(projectName, baseDir = process.cwd()) {
+    return path.resolve(baseDir, "dags", projectName)
+}
+
+export function checkDagsDirectory(baseDir = process.cwd()) {
+    const dagsPath = path.resolve(baseDir, "dags")
+    return fs.existsSync(dagsPath) && fs.lstatSync(dagsPath).isDirectory()
+}
+
+export async function getExistingProjects(baseDir = process.cwd()) {
+    const dagsPath = path.resolve(baseDir, "dags")
+    if (!fs.existsSync(dagsPath)) return []
+    const dirents = await fs.readdir(dagsPath, { withFileTypes: true })
+    return dirents
+        .filter((dirent) => dirent.isDirectory())
+        .map((dirent) => dirent.name)
+}

package/src/lib/filesystem.js

@@ -0,0 +1,45 @@
+import fs from "fs-extra"
+import path from "path"
+
+export async function replaceInFiles(dir, replacements) {
+    const files = await getFiles(dir)
+
+    for (const file of files) {
+        // Only process text files we care about, e.g., .py, .md, .txt, .json
+        // For a DAG template, .py is critical.
+        if (shouldProcess(file)) {
+            let content = await fs.readFile(file, "utf8")
+            let changed = false
+
+            for (const [key, value] of Object.entries(replacements)) {
+                // Replace all occurrences
+                const regex = new RegExp(key, "g")
+                if (regex.test(content)) {
+                    content = content.replace(regex, value)
+                    changed = true
+                }
+            }
+
+            if (changed) {
+                await fs.writeFile(file, content, "utf8")
+            }
+        }
+    }
+}
+
+async function getFiles(dir) {
+    const dirents = await fs.readdir(dir, { withFileTypes: true })
+    const files = await Promise.all(
+        dirents.map((dirent) => {
+            const res = path.resolve(dir, dirent.name)
+            return dirent.isDirectory() ? getFiles(res) : res
+        }),
+    )
+    return Array.prototype.concat(...files)
+}
+
+function shouldProcess(filePath) {
+    const ext = path.extname(filePath)
+    // Add extensions you want to scan for replacement
+    return [".py", ".md", ".json", ".txt", ".yaml", ".yml"].includes(ext)
+}

package/src/lib/git.js

@@ -0,0 +1,13 @@
+import { execa } from "execa"
+
+export async function initGitRepo(dir) {
+    try {
+        await execa("git", ["init"], { cwd: dir })
+        return true
+    } catch (error) {
+        // Silently fail if git init fails, or throw if critical.
+        // For a scaffolding tool, it's usually better to warn than crash.
+        // But for now, we'll just throw so the caller can decide.
+        throw new Error(`Failed to initialize git repository: ${error.message}`)
+    }
+}

package/src/main.js

@@ -0,0 +1,71 @@
+import { initCommand } from "./commands/init.js"
+import { printHelp, printVersion } from "./ui/display.js"
+import chalk from "chalk"
+import fs from "fs-extra"
+
+// Enforce Node Version Check
+const requiredVersion = 20
+const currentVersion = process.versions.node.split(".")[0]
+if (currentVersion < requiredVersion) {
+    console.error(
+        chalk.red(
+            `Error: Node.js version ${requiredVersion}+ is required. You are running version ${process.versions.node}.`,
+        ),
+    )
+    process.exit(1)
+}
+
+// Read package.json for version
+const pkg = fs.readJsonSync(new URL("../package.json", import.meta.url))
+
+export async function main() {
+    const args = process.argv.slice(2)
+    // Basic argument parsing
+    let command = args[0]
+    let airflowPath
+
+    // Check for --airflow-path argument anywhere
+    const pathIndex = args.indexOf("--airflow-path")
+    if (pathIndex !== -1 && args[pathIndex + 1]) {
+        airflowPath = args[pathIndex + 1]
+        // If the command was actually the path flag (e.g. "airframe --airflow-path ... init")
+        // we need to find the real command.
+        // Simple heuristic: First arg that isn't a flag or a value of a flag.
+        // For now, let's just stick to typical usage or basic filter.
+        if (command === "--airflow-path") {
+            // Try to find the real command
+            const nonFlagArgs = args.filter(
+                (arg, idx) =>
+                    arg !== "--airflow-path" &&
+                    args[idx - 1] !== "--airflow-path",
+            )
+            command = nonFlagArgs[0]
+        }
+    }
+
+    // Handle flags anywhere
+    if (args.includes("--version") || args.includes("-v")) {
+        printVersion(pkg.version)
+        process.exit(0)
+    }
+
+    if (args.includes("--help") || args.includes("-h")) {
+        printHelp()
+        process.exit(0)
+    }
+
+    // If no command is provided, default to "init"
+    if (!command) {
+        command = "init"
+    }
+
+    // Handle commands
+    switch (command) {
+        case "init":
+            await initCommand({ airflowPath })
+            break
+        default:
+            printHelp()
+            break
+    }
+}

package/src/scaffolder.js

@@ -0,0 +1,46 @@
+import tiged from "tiged"
+import { replaceInFiles } from "./lib/filesystem.js"
+
+export async function scaffoldProject({ template, targetDir }) {
+    // template.id should be the branch name, tag, or value
+    const uri = `hraza01/airframe-templates#${template.branch || template.id || template.value}`
+
+    const emitter = tiged(uri, {
+        disableCache: true,
+        force: true,
+        verbose: false,
+    })
+
+    try {
+        await emitter.clone(targetDir)
+    } catch (error) {
+        throw new Error(`Failed to scaffold project: ${error.message}`)
+    }
+}
+
+export async function customizeProject({
+    targetDir,
+    projectName,
+    dagAuthor,
+    template,
+}) {
+    try {
+        await replaceInFiles(targetDir, {
+            __PROJECT_NAME__: projectName,
+            __DAG_ID__: projectName,
+            __DAG_AUTHOR__: dagAuthor,
+        })
+
+        // Cleanup for Advanced template
+        if (template && template.label === "Advanced") {
+            await replaceInFiles(targetDir, {
+                "# fmt: off\\n": "",
+                "# isort: off\\n": "",
+                "# isort: on\\n": "",
+                "# fmt: on\\n": "",
+            })
+        }
+    } catch (error) {
+        throw new Error(`Failed to customize project files: ${error.message}`)
+    }
+}

package/src/ui/display.js

@@ -0,0 +1,101 @@
+import { intro, outro, log } from "@clack/prompts"
+import chalk from "chalk"
+import terminalLink from "terminal-link"
+import figlet from "figlet"
+
+export function printBanner() {
+    console.clear()
+    console.log(
+        chalk.cyan(figlet.textSync("AIRFRAME", { font: "ANSI Shadow" })),
+    )
+    const authorLink = terminalLink("@hraza01", "https://github.com/hraza01")
+    console.log(
+        chalk.dim.blackBright(
+            `                                         — created by ${authorLink}\n`,
+        ),
+    )
+}
+
+export function showWelcome(workingDir) {
+    printBanner()
+    intro(chalk.bold.cyanBright("The blueprint for scalable Airflow DAGs"))
+
+    // Explicit note about Airflow support
+    log.info(chalk.dim("Supports Apache Airflow 2.0 (2.8 or higher)"))
+    logWorkingDir(workingDir)
+    log.warn(chalk.dim("Press CTRL+C to exit the Airframe CLI.\n"))
+}
+
+export function logWorkingDir(dir = process.cwd()) {
+    log.warn(
+        `You are about to initialize an Airframe project in this directory:\n${chalk.dim(dir)}`,
+    )
+}
+
+export function showCompletion({ targetDir }) {
+    outro(chalk.green.bold(`Airframe project initialized.`))
+
+    console.log(
+        chalk.bold.magenta(
+            `Project directory: ${chalk.underline(targetDir)}\n`,
+        ),
+    )
+
+    console.log(chalk.bold.blueBright("Happy Orchestrating!"))
+}
+
+export function printHelp() {
+    printBanner()
+
+    console.log(chalk.bold("Usage:"))
+    console.log("  npx create-airframe-dag@latest [options]\n")
+
+    console.log(chalk.bold("Options:"))
+    console.log(
+        "  --airflow-path <path>   Specify the path to your Airflow root directory",
+    )
+    console.log("  -v, --version           Show version number")
+    console.log("  -h, --help              Show this help message\n")
+
+    console.log(chalk.dim("Examples:"))
+    console.log("  npx create-airframe-dag@latest")
+    console.log("  npx create-airframe-dag@latest --version\n")
+}
+
+export function printVersion(version) {
+    console.log(chalk.bold(`v${version}`))
+}
+
+// Helper to wrap text with indentation for subsequent lines
+function formatDescription(text, width, indent) {
+    if (!text) return ""
+    const words = text.split(" ")
+    let lines = []
+    let currentLine = words[0]
+
+    for (let i = 1; i < words.length; i++) {
+        if (currentLine.length + 1 + words[i].length <= width) {
+            currentLine += " " + words[i]
+        } else {
+            lines.push(currentLine)
+            currentLine = words[i]
+        }
+    }
+    lines.push(currentLine)
+
+    return lines.join("\n" + indent)
+}
+
+export function displayTemplates(templates) {
+    log.info(chalk.bold("Available Templates"))
+
+    const maxLabelLen = Math.max(...templates.map((t) => t.label.length))
+
+    templates.forEach((t) => {
+        const pad = " ".repeat(maxLabelLen - t.label.length)
+        const indent = " ".repeat(maxLabelLen + 4)
+        const desc = formatDescription(t.description || "", 45, indent)
+
+        log.message(`${chalk.cyan(t.label)}${pad}    ${chalk.dim(desc)}`)
+    })
+}

package/src/ui/prompts.js

@@ -0,0 +1,83 @@
+import { text, select, isCancel, cancel } from "@clack/prompts"
+import ora from "ora"
+
+export function createSpinner() {
+    return ora()
+}
+
+export async function askForProjectDetails(existingProjects = []) {
+    const projectName = await text({
+        message: "What is the name of your Airflow DAG?",
+        hint: "This will be the name of the DAG folder and the DAG ID.",
+        placeholder: "my_dag_project",
+        validate(value) {
+            if (!value || value.trim().length === 0)
+                return "Project name is required!"
+            if (/[^a-zA-Z_]/.test(value))
+                return "Project name should only contain letters and underscores (no numbers, dashes, or spaces)."
+            if (
+                existingProjects.some(
+                    (p) => p.toLowerCase() === value.toLowerCase(),
+                )
+            )
+                return `Project "${value}" already exists in dags/ directory.`
+        },
+    })
+
+    if (isCancel(projectName)) {
+        cancel("Operation cancelled.")
+        process.exit(0)
+    }
+
+    const dagAuthor = await text({
+        message: "Who is the author of this Airflow DAG?",
+        placeholder: "Anakin Skywalker",
+        validate(value) {
+            if (!value || value.trim().length === 0)
+                return "Author name is required!"
+            if (value.toLowerCase() === "airflow")
+                return "Author name cannot be 'airflow'."
+            if (/[^a-zA-Z\s_]/.test(value))
+                return "Author name must only contain letters, spaces, and underscores."
+        },
+    })
+
+    if (isCancel(dagAuthor)) {
+        cancel("Operation cancelled.")
+        process.exit(0)
+    }
+
+    return { projectName, dagAuthor }
+}
+
+export async function selectTemplate(templates) {
+    // 1. Find max label length for padding
+    const maxLabelLen = Math.max(...templates.map((t) => t.label.length))
+
+    const options = templates.map((t) => {
+        // 2. Pad label
+        const pad = " ".repeat(maxLabelLen - t.label.length)
+        const label = `${t.label}${pad}`
+
+        // 3. Use standard hint property (accepting brackets to keep UI clean)
+        const hint = t.hint || ""
+
+        return {
+            value: t,
+            label: label,
+            hint: hint,
+        }
+    })
+
+    const template = await select({
+        message: "Select a template to scaffold:",
+        options: options,
+    })
+
+    if (isCancel(template)) {
+        cancel("Operation cancelled.")
+        process.exit(0)
+    }
+
+    return template
+}