@jacobknightley/fabric-format 0.0.1

package/dist/index.js

@@ -0,0 +1,41 @@
+/**
+ * sparkfmt - Spark SQL & Python Formatter
+ *
+ * A unified formatter for Spark SQL and Python code, designed for
+ * Microsoft Fabric notebooks and CI/CD pipelines.
+ *
+ * Architecture:
+ * - formatters/sql/: Core SQL formatting (ANTLR grammar-driven)
+ * - formatters/python/: Python formatting via Ruff WASM
+ * - notebook-formatter.ts: Fabric notebook handling
+ */
+// ============================================================================
+// SQL Formatter (Core API)
+// ============================================================================
+export { formatSql, needsFormatting } from './formatters/sparksql/index.js';
+// ============================================================================
+// Language Formatters (Extensible)
+// ============================================================================
+export { 
+// Registry
+getFormatterRegistry, detectLanguage, 
+// SQL
+SqlFormatter, getSqlFormatter, isSqlCode, 
+// Python
+PythonFormatter, getPythonFormatter, isPythonCode, } from './formatters/index.js';
+// ============================================================================
+// Cell Formatter (Low-level API)
+// ============================================================================
+export { formatCell, formatCellSync, initializePythonFormatter, isPythonFormatterReady, } from './cell-formatter.js';
+// ============================================================================
+// Notebook Formatter (High-level API)
+// ============================================================================
+export { parseNotebook, formatNotebook, } from './notebook-formatter.js';
+// ============================================================================
+// Configuration (Python/Ruff)
+// ============================================================================
+export { DEFAULT_RUFF_CONFIG, RUFF_WASM_CONFIG, } from './formatters/python/index.js';
+// ============================================================================
+// Format Directives (Spark SQL)
+// ============================================================================
+export { hasFormatOff, detectCollapseDirectives, hasCollapseDirective } from './formatters/sparksql/index.js';

package/dist/notebook-formatter.d.ts

@@ -0,0 +1,107 @@
+/**
+ * Notebook Formatter
+ *
+ * High-level API for parsing and formatting Microsoft Fabric notebook files.
+ * Handles the notebook structure, cell extraction, and comment wrapper management.
+ *
+ * For low-level cell formatting (when you already know the cell type),
+ * use the cell-formatter module directly.
+ *
+ * File formats:
+ * - .py files (Python/PySpark) with `# MAGIC` prefix for Spark SQL cells
+ * - .scala files (Scala/Spark) with `// MAGIC` prefix for Spark SQL cells
+ * - .r files (R/SparkR) with `# MAGIC` prefix for Spark SQL cells
+ * - .sql files (Spark SQL) with `-- MAGIC` prefix OR raw SQL
+ *
+ * Cell structure (Python example):
+ * ```
+ * # CELL ********************
+ *
+ * # MAGIC %%sql
+ * # MAGIC SELECT *
+ * # MAGIC FROM table
+ *
+ * # METADATA ********************
+ * # META {
+ * # META   "language": "sparksql",
+ * # META   "language_group": "synapse_pyspark"
+ * # META }
+ * ```
+ */
+export { formatCell, formatCellSync, initializePythonFormatter, isPythonFormatterReady, type FormatCellResult, type CellType, } from './cell-formatter.js';
+/** Language-specific comment prefixes */
+interface LanguageConfig {
+    fabricHeader: string;
+    cellMarker: string;
+    metadataMarker: string;
+    magicPrefix: string;
+    magicSqlCommand: string;
+    emptyMagic: string;
+    /** If true, cells can contain raw SQL without MAGIC prefix */
+    supportsRawSql: boolean;
+    /** Default cell language when no magic command present */
+    defaultLanguage: string;
+}
+/** Represents a cell in a Fabric notebook */
+export interface NotebookCell {
+    /** Line index (0-based) where the cell content starts (after CELL marker) */
+    contentStartLine: number;
+    /** Line index (0-based) where the cell content ends (before METADATA) */
+    contentEndLine: number;
+    /** The extracted content (without MAGIC prefixes) */
+    content: string;
+    /** Original lines including MAGIC prefixes */
+    originalLines: string[];
+    /** Detected language of this cell (from METADATA) */
+    language: string;
+    /** Whether this is a MAGIC-prefixed cell */
+    isMagicCell: boolean;
+    /** Whether this is a raw cell (no MAGIC prefix) */
+    isRawCell: boolean;
+    /** The magic command used in the cell (e.g., 'sql', 'pyspark', 'configure'), or null if no magic */
+    magicCommand: string | null;
+}
+/** Represents a parsed Fabric notebook */
+export interface FabricNotebook {
+    /** Whether this file is a Fabric notebook */
+    isFabricNotebook: boolean;
+    /** All detected cells */
+    cells: NotebookCell[];
+    /** Original file content split into lines */
+    lines: string[];
+    /** The detected language config */
+    config: LanguageConfig | null;
+}
+/** Statistics from formatting operation */
+export interface FormatStats {
+    sparkSqlCellsFormatted: number;
+    pythonCellsFormatted: number;
+    cellsSkipped: number;
+    errors: string[];
+}
+/**
+ * Parse a Fabric notebook file into cells.
+ * @param content The file content
+ * @param fileExtension The file extension (e.g., '.py', '.sql', '.scala', '.r')
+ */
+export declare function parseNotebook(content: string, fileExtension: string): FabricNotebook;
+/**
+ * Format all cells in a Fabric notebook.
+ *
+ * This is the high-level API for formatting entire notebook files.
+ * It parses the notebook structure, extracts cells, formats them using
+ * the low-level formatCell API, and reassembles with proper comment wrappers.
+ *
+ * @param content The file content
+ * @param fileExtension The file extension (e.g., '.py', '.sql', '.scala', '.r')
+ * @param options Formatting options
+ * @returns Object with formatted content and statistics
+ */
+export declare function formatNotebook(content: string, fileExtension: string, options?: {
+    formatSql?: boolean;
+    formatPython?: boolean;
+    configPath?: string;
+}): Promise<{
+    content: string;
+    stats: FormatStats;
+}>;

package/dist/notebook-formatter.js

@@ -0,0 +1,424 @@
+/**
+ * Notebook Formatter
+ *
+ * High-level API for parsing and formatting Microsoft Fabric notebook files.
+ * Handles the notebook structure, cell extraction, and comment wrapper management.
+ *
+ * For low-level cell formatting (when you already know the cell type),
+ * use the cell-formatter module directly.
+ *
+ * File formats:
+ * - .py files (Python/PySpark) with `# MAGIC` prefix for Spark SQL cells
+ * - .scala files (Scala/Spark) with `// MAGIC` prefix for Spark SQL cells
+ * - .r files (R/SparkR) with `# MAGIC` prefix for Spark SQL cells
+ * - .sql files (Spark SQL) with `-- MAGIC` prefix OR raw SQL
+ *
+ * Cell structure (Python example):
+ * ```
+ * # CELL ********************
+ *
+ * # MAGIC %%sql
+ * # MAGIC SELECT *
+ * # MAGIC FROM table
+ *
+ * # METADATA ********************
+ * # META {
+ * # META   "language": "sparksql",
+ * # META   "language_group": "synapse_pyspark"
+ * # META }
+ * ```
+ */
+import { formatCell, initializePythonFormatter, } from './cell-formatter.js';
+// Re-export cell-formatter types and functions for convenience
+export { formatCell, formatCellSync, initializePythonFormatter, isPythonFormatterReady, } from './cell-formatter.js';
+const PYTHON_CONFIG = {
+    fabricHeader: '# Fabric notebook source',
+    cellMarker: '# CELL ********************',
+    metadataMarker: '# METADATA ********************',
+    magicPrefix: '# MAGIC ',
+    magicSqlCommand: '# MAGIC %%sql',
+    emptyMagic: '# MAGIC',
+    supportsRawSql: false,
+    defaultLanguage: 'python',
+};
+const SCALA_CONFIG = {
+    fabricHeader: '// Fabric notebook source',
+    cellMarker: '// CELL ********************',
+    metadataMarker: '// METADATA ********************',
+    magicPrefix: '// MAGIC ',
+    magicSqlCommand: '// MAGIC %%sql',
+    emptyMagic: '// MAGIC',
+    supportsRawSql: false,
+    defaultLanguage: 'scala',
+};
+const SPARKSQL_CONFIG = {
+    fabricHeader: '-- Fabric notebook source',
+    cellMarker: '-- CELL ********************',
+    metadataMarker: '-- METADATA ********************',
+    magicPrefix: '-- MAGIC ',
+    magicSqlCommand: '-- MAGIC %%sql',
+    emptyMagic: '-- MAGIC',
+    supportsRawSql: true,
+    defaultLanguage: 'sparksql',
+};
+const R_CONFIG = {
+    ...PYTHON_CONFIG, // R uses same comment syntax as Python
+    fabricHeader: '# Fabric notebook source', // Check if R has different header
+    defaultLanguage: 'r',
+};
+// ============================================================================
+// INTERNAL UTILITIES
+// ============================================================================
+// ============================================================================
+// INTERNAL UTILITIES
+// ============================================================================
+/**
+ * Detect line ending style.
+ */
+function detectLineEnding(content) {
+    if (content.includes('\r\n'))
+        return '\r\n';
+    return '\n';
+}
+/**
+ * Get language config based on file extension.
+ * The file extension determines the default language and comment syntax.
+ */
+function getLanguageConfig(fileExtension) {
+    const ext = fileExtension.toLowerCase();
+    switch (ext) {
+        case '.py':
+            return PYTHON_CONFIG;
+        case '.scala':
+            return SCALA_CONFIG;
+        case '.sql':
+            return SPARKSQL_CONFIG;
+        case '.r':
+            return R_CONFIG;
+        default:
+            return null;
+    }
+}
+/**
+ * Validate that the file content is a Fabric notebook.
+ * Checks that the first line matches the expected header for the config.
+ */
+function isFabricNotebookContent(firstLine, config) {
+    return firstLine.trim().startsWith(config.fabricHeader);
+}
+/**
+ * Extract the magic command from cell lines (e.g., 'sql', 'pyspark', 'configure').
+ * Returns null if no magic command is present.
+ */
+function extractMagicCommand(lines, config) {
+    for (const line of lines) {
+        const trimmed = line.trim();
+        // Look for magic command pattern: # MAGIC %%<command>
+        const magicPrefix = config.magicPrefix + '%%';
+        if (trimmed.startsWith(magicPrefix)) {
+            // Extract command name (everything after %% until space or end)
+            const afterPrefix = trimmed.slice(magicPrefix.length);
+            const match = afterPrefix.match(/^(\w+)/);
+            if (match) {
+                return match[1].toLowerCase();
+            }
+        }
+    }
+    return null;
+}
+/**
+ * Parse the METADATA block to extract the language.
+ * Returns the language string or null if not found.
+ */
+function parseMetadataLanguage(lines, startLine, config) {
+    // Find the METADATA marker
+    let i = startLine;
+    while (i < lines.length && lines[i].trim() !== config.metadataMarker) {
+        i++;
+    }
+    if (i >= lines.length)
+        return null;
+    // Parse the META JSON block
+    // Format: # META { ... "language": "sparksql" ... }
+    const metaPrefix = config.magicPrefix.replace('MAGIC', 'META').trim();
+    let jsonContent = '';
+    i++; // Skip the METADATA marker
+    while (i < lines.length) {
+        const trimmed = lines[i].trim();
+        if (trimmed === config.cellMarker || trimmed === config.metadataMarker) {
+            break;
+        }
+        // Extract content after # META prefix
+        if (trimmed.startsWith(metaPrefix)) {
+            jsonContent += trimmed.slice(metaPrefix.length).trim();
+        }
+        i++;
+    }
+    // Try to parse and extract language
+    try {
+        const meta = JSON.parse(jsonContent);
+        if (meta && typeof meta.language === 'string') {
+            return meta.language;
+        }
+    }
+    catch {
+        // JSON parsing failed, ignore
+    }
+    return null;
+}
+/**
+ * Detect the language of a cell based on magic commands.
+ * @deprecated Use parseMetadataLanguage instead - kept for backward compatibility
+ */
+function detectCellLanguage(lines, config) {
+    for (const line of lines) {
+        const trimmed = line.trim();
+        // Check for magic commands
+        if (trimmed === config.magicSqlCommand || trimmed.startsWith(config.magicPrefix + '%%sql')) {
+            return 'sparksql';
+        }
+        if (trimmed === config.magicPrefix + '%%python' || trimmed.startsWith(config.magicPrefix + '%%python ')) {
+            return 'python';
+        }
+        if (trimmed === config.magicPrefix + '%%pyspark' || trimmed.startsWith(config.magicPrefix + '%%pyspark ')) {
+            return 'python'; // PySpark is Python
+        }
+        if (trimmed === config.magicPrefix + '%%scala' || trimmed.startsWith(config.magicPrefix + '%%scala ')) {
+            return 'scala';
+        }
+        if (trimmed === config.magicPrefix + '%%r' || trimmed === config.magicPrefix + '%%R') {
+            return 'r';
+        }
+        // First non-empty, non-magic line determines if it's a raw cell
+        if (trimmed !== '' && !trimmed.startsWith(config.magicPrefix.trim())) {
+            return config.defaultLanguage;
+        }
+    }
+    return config.defaultLanguage;
+}
+/**
+ * Strip MAGIC prefix from a line.
+ */
+function stripMagicPrefix(line, config) {
+    if (line.startsWith(config.magicPrefix)) {
+        return line.slice(config.magicPrefix.length);
+    }
+    if (line.trim() === config.emptyMagic) {
+        return '';
+    }
+    return line;
+}
+/**
+ * Add MAGIC prefix to lines.
+ */
+function addMagicPrefix(content, config) {
+    return content.split(/\r?\n/).map(line => {
+        if (line === '')
+            return config.emptyMagic;
+        return config.magicPrefix + line;
+    });
+}
+/**
+ * Parse a Fabric notebook file into cells.
+ * @param content The file content
+ * @param fileExtension The file extension (e.g., '.py', '.sql', '.scala', '.r')
+ */
+export function parseNotebook(content, fileExtension) {
+    const lines = content.split(/\r?\n/);
+    const result = {
+        isFabricNotebook: false,
+        cells: [],
+        lines,
+        config: null,
+    };
+    if (lines.length === 0)
+        return result;
+    const config = getLanguageConfig(fileExtension);
+    if (!config)
+        return result;
+    // Validate it's actually a Fabric notebook
+    if (!isFabricNotebookContent(lines[0], config))
+        return result;
+    result.isFabricNotebook = true;
+    result.config = config;
+    let i = 0;
+    while (i < lines.length) {
+        if (lines[i].trim() === config.cellMarker) {
+            const cellStartLine = i + 1;
+            // Skip empty lines after CELL marker
+            let j = cellStartLine;
+            while (j < lines.length && lines[j].trim() === '') {
+                j++;
+            }
+            // Find end of cell
+            let cellEndLine = j;
+            while (cellEndLine < lines.length) {
+                const trimmed = lines[cellEndLine].trim();
+                if (trimmed === config.metadataMarker || trimmed === config.cellMarker) {
+                    break;
+                }
+                cellEndLine++;
+            }
+            // Back up past trailing empty lines
+            let actualEndLine = cellEndLine - 1;
+            while (actualEndLine >= j && lines[actualEndLine].trim() === '') {
+                actualEndLine--;
+            }
+            if (actualEndLine >= j) {
+                const originalLines = lines.slice(j, actualEndLine + 1);
+                // Get language from METADATA block (authoritative source)
+                const metadataLanguage = parseMetadataLanguage(lines, cellEndLine, config);
+                // Map metadata language to our internal language names
+                let language;
+                if (metadataLanguage === 'sparksql') {
+                    language = 'sparksql';
+                }
+                else if (metadataLanguage === 'python' || metadataLanguage === 'pyspark') {
+                    language = 'python';
+                }
+                else if (metadataLanguage === 'scala') {
+                    language = 'scala';
+                }
+                else if (metadataLanguage === 'r' || metadataLanguage === 'R') {
+                    language = 'r';
+                }
+                else if (metadataLanguage) {
+                    language = metadataLanguage;
+                }
+                else {
+                    // Fallback to magic-based detection if no metadata
+                    language = detectCellLanguage(originalLines, config);
+                }
+                // Extract magic command (e.g., 'sql', 'pyspark', 'configure')
+                const magicCommand = extractMagicCommand(originalLines, config);
+                // Check if it's a MAGIC cell
+                const isMagicCell = originalLines.some(l => l.trim().startsWith(config.magicPrefix.trim()));
+                // Extract content
+                let content;
+                let contentStartLine = j;
+                if (isMagicCell) {
+                    // Skip the magic command line (%%sql, %%python, etc.)
+                    const magicCommandIndex = originalLines.findIndex(l => l.trim().startsWith(config.magicPrefix + '%%'));
+                    if (magicCommandIndex >= 0) {
+                        const contentLines = originalLines.slice(magicCommandIndex + 1);
+                        content = contentLines.map(l => stripMagicPrefix(l, config)).join('\n');
+                        contentStartLine = j + magicCommandIndex + 1;
+                    }
+                    else {
+                        content = originalLines.map(l => stripMagicPrefix(l, config)).join('\n');
+                    }
+                }
+                else {
+                    content = originalLines.join('\n');
+                }
+                result.cells.push({
+                    contentStartLine: contentStartLine,
+                    contentEndLine: actualEndLine,
+                    content,
+                    originalLines,
+                    language,
+                    isMagicCell,
+                    isRawCell: !isMagicCell,
+                    magicCommand,
+                });
+            }
+            i = cellEndLine;
+            continue;
+        }
+        i++;
+    }
+    return result;
+}
+/**
+ * Replace a cell's content in the file.
+ */
+function replaceCell(fileContent, cell, formattedContent, config, lineEnding) {
+    const lines = fileContent.split(/\r?\n/);
+    let newLines;
+    if (cell.isMagicCell) {
+        newLines = addMagicPrefix(formattedContent, config);
+    }
+    else {
+        newLines = formattedContent.split(/\r?\n/);
+    }
+    const before = lines.slice(0, cell.contentStartLine);
+    const after = lines.slice(cell.contentEndLine + 1);
+    return [...before, ...newLines, ...after].join(lineEnding);
+}
+/**
+ * Format all cells in a Fabric notebook.
+ *
+ * This is the high-level API for formatting entire notebook files.
+ * It parses the notebook structure, extracts cells, formats them using
+ * the low-level formatCell API, and reassembles with proper comment wrappers.
+ *
+ * @param content The file content
+ * @param fileExtension The file extension (e.g., '.py', '.sql', '.scala', '.r')
+ * @param options Formatting options
+ * @returns Object with formatted content and statistics
+ */
+export async function formatNotebook(content, fileExtension, options) {
+    const formatSparkSqlCells = options?.formatSql ?? true;
+    const formatPythonCells = options?.formatPython ?? true;
+    const stats = {
+        sparkSqlCellsFormatted: 0,
+        pythonCellsFormatted: 0,
+        cellsSkipped: 0,
+        errors: [],
+    };
+    const notebook = parseNotebook(content, fileExtension);
+    if (!notebook.isFabricNotebook || notebook.cells.length === 0 || !notebook.config) {
+        return { content, stats };
+    }
+    const lineEnding = detectLineEnding(content);
+    // Initialize Python formatter if needed
+    if (formatPythonCells) {
+        try {
+            await initializePythonFormatter();
+        }
+        catch (error) {
+            stats.errors.push(`Python formatter init failed: ${error}`);
+        }
+    }
+    // Process cells in reverse order (to preserve line numbers)
+    let result = content;
+    const cellsReversed = [...notebook.cells].reverse();
+    for (const cell of cellsReversed) {
+        // Determine if this cell should be formatted based on language and magic command
+        // Spark SQL cells: format only if %%sql magic or no magic command
+        // Python cells: format only if %%pyspark magic or no magic command
+        const magicCmd = cell.magicCommand;
+        const shouldFormatSparkSql = cell.language === 'sparksql' && formatSparkSqlCells &&
+            (magicCmd === null || magicCmd === 'sql');
+        const shouldFormatPython = cell.language === 'python' && formatPythonCells &&
+            (magicCmd === null || magicCmd === 'pyspark');
+        if (shouldFormatSparkSql) {
+            // Format using low-level API (cell.content is already stripped of MAGIC prefixes)
+            const formatResult = formatCell(cell.content, 'sparksql');
+            if (formatResult.changed) {
+                // replaceCell will add back MAGIC prefixes if needed
+                result = replaceCell(result, cell, formatResult.formatted, notebook.config, lineEnding);
+                stats.sparkSqlCellsFormatted++;
+            }
+            if (formatResult.error) {
+                stats.errors.push(formatResult.error);
+            }
+        }
+        else if (shouldFormatPython) {
+            // Format using low-level API (cell.content is already stripped of MAGIC prefixes)
+            const formatResult = formatCell(cell.content, 'python');
+            if (formatResult.changed) {
+                // replaceCell will add back MAGIC prefixes if needed
+                result = replaceCell(result, cell, formatResult.formatted, notebook.config, lineEnding);
+                stats.pythonCellsFormatted++;
+            }
+            if (formatResult.error) {
+                stats.errors.push(formatResult.error);
+            }
+        }
+        else {
+            stats.cellsSkipped++;
+        }
+    }
+    return { content: result, stats };
+}

package/package.json

@@ -0,0 +1,63 @@
+{
+  "name": "@jacobknightley/fabric-format",
+  "version": "0.0.1",
+  "type": "module",
+  "description": "A fast, opinionated formatter for Microsoft Fabric notebooks with Spark SQL and Python support",
+  "main": "dist/index.js",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "types": "dist/index.d.ts",
+  "bin": {
+    "fabfmt": "dist/cli.js"
+  },
+  "files": [
+    "dist/**/*.js",
+    "dist/**/*.d.ts",
+    "!dist/tests/**",
+    "README.md"
+  ],
+  "keywords": [
+    "fabric",
+    "microsoft-fabric",
+    "spark",
+    "sql",
+    "python",
+    "pyspark",
+    "formatter",
+    "notebook",
+    "delta-lake",
+    "antlr",
+    "ruff"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/jacobknightley/fabric-format.git"
+  },
+  "homepage": "https://github.com/jacobknightley/fabric-format#readme",
+  "bugs": {
+    "url": "https://github.com/jacobknightley/fabric-format/issues"
+  },
+  "author": "Jacob Knightley",
+  "license": "MIT",
+  "scripts": {
+    "build:antlr": "python .build/build_antlr_js.py",
+    "build:ts": "tsc",
+    "build:readme": "node -e \"require('fs').copyFileSync('../../README.md', './README.md')\"",
+    "build": "npm run build:readme && npm run build:antlr && npm run build:ts",
+    "test": "node dist/tests/index.js",
+    "test:verbose": "node dist/tests/index.js --verbose",
+    "format": "node dist/cli.js"
+  },
+  "devDependencies": {
+    "@types/node": "^20.10.0",
+    "typescript": "^5.3.0"
+  },
+  "dependencies": {
+    "@astral-sh/ruff-wasm-web": "^0.14.10",
+    "antlr4": "^4.13.2"
+  }
+}