@luquimbo/bi-superpowers 1.0.0

package/library/snippets/power-query/connections.md

@@ -0,0 +1,434 @@
+# Data Source Connections
+
+## Overview
+Power Query patterns for connecting to various data sources including databases, cloud services, APIs, and files.
+
+---
+
+## SQL Server
+
+### Basic Connection
+```m
+let
+    Source = Sql.Database("server-name", "database-name")
+in
+    Source
+```
+
+### With Native Query
+```m
+let
+    Source = Sql.Database("server-name", "database-name", [
+        Query = "
+            SELECT
+                CustomerID,
+                CustomerName,
+                Email,
+                CreatedDate
+            FROM dbo.Customers
+            WHERE IsActive = 1
+        "
+    ])
+in
+    Source
+```
+
+### With Parameters
+```m
+let
+    Source = Sql.Database(param_ServerName, param_DatabaseName, [
+        Query = "
+            SELECT * FROM dbo.Sales
+            WHERE OrderDate >= '" & Date.ToText(param_StartDate, "yyyy-MM-dd") & "'
+        ",
+        CommandTimeout = #duration(0, 0, 10, 0)  // 10 minutes
+    ])
+in
+    Source
+```
+
+### Navigation to Table
+```m
+let
+    Source = Sql.Database("server-name", "database-name"),
+    Sales = Source{[Schema="dbo", Item="Sales"]}[Data]
+in
+    Sales
+```
+
+---
+
+## SharePoint
+
+### SharePoint List
+```m
+let
+    Source = SharePoint.Tables(
+        "https://company.sharepoint.com/sites/SiteName",
+        [Implementation = "2.0", ViewMode = "All"]
+    ),
+    ListData = Source{[Title="ListName"]}[Items],
+    // Remove SharePoint metadata columns
+    RemovedColumns = Table.SelectColumns(ListData, {"Column1", "Column2", "Column3"})
+in
+    RemovedColumns
+```
+
+### SharePoint Folder (Files)
+```m
+let
+    Source = SharePoint.Files(
+        "https://company.sharepoint.com/sites/SiteName",
+        [ApiVersion = 15]
+    ),
+    // Filter to specific folder
+    FilteredFolder = Table.SelectRows(Source, each
+        Text.Contains([Folder Path], "/Documents/Reports/")
+    ),
+    // Filter to Excel files
+    FilteredFiles = Table.SelectRows(FilteredFolder, each
+        Text.EndsWith([Name], ".xlsx")
+    )
+in
+    FilteredFiles
+```
+
+### SharePoint Excel File
+```m
+let
+    Source = SharePoint.Files(
+        "https://company.sharepoint.com/sites/SiteName",
+        [ApiVersion = 15]
+    ),
+    // Find specific file
+    FileRow = Table.SelectRows(Source, each [Name] = "SalesData.xlsx"){0},
+    // Load Excel content
+    ExcelContent = Excel.Workbook(FileRow[Content], true, true),
+    // Get specific sheet
+    Sheet1 = ExcelContent{[Item="Sheet1", Kind="Sheet"]}[Data]
+in
+    Sheet1
+```
+
+---
+
+## REST API / Web Services
+
+### Basic JSON API
+```m
+let
+    Source = Json.Document(
+        Web.Contents("https://api.example.com/data")
+    ),
+    ToTable = Table.FromRecords(Source)
+in
+    ToTable
+```
+
+### With Authentication Header
+```m
+let
+    Source = Json.Document(
+        Web.Contents(
+            "https://api.example.com/data",
+            [
+                Headers = [
+                    #"Authorization" = "Bearer " & param_APIToken,
+                    #"Content-Type" = "application/json"
+                ]
+            ]
+        )
+    )
+in
+    Source
+```
+
+### With Query Parameters
+```m
+let
+    Source = Json.Document(
+        Web.Contents(
+            "https://api.example.com/data",
+            [
+                Query = [
+                    startDate = Date.ToText(param_StartDate, "yyyy-MM-dd"),
+                    endDate = Date.ToText(param_EndDate, "yyyy-MM-dd"),
+                    limit = "1000",
+                    page = "1"
+                ]
+            ]
+        )
+    )
+in
+    Source
+```
+
+### Pagination Pattern
+```m
+let
+    // Function to get one page
+    GetPage = (pageNum as number) as table =>
+        let
+            Source = Json.Document(
+                Web.Contents(
+                    "https://api.example.com/data",
+                    [Query = [page = Text.From(pageNum), limit = "100"]]
+                )
+            ),
+            ToTable = Table.FromRecords(Source[data])
+        in
+            ToTable,
+
+    // Get all pages
+    PageCount = 10,  // Or dynamically determine
+    Pages = List.Generate(
+        () => [Page = 1, Data = GetPage(1)],
+        each [Page] <= PageCount,
+        each [Page = [Page] + 1, Data = GetPage([Page] + 1)],
+        each [Data]
+    ),
+    Combined = Table.Combine(Pages)
+in
+    Combined
+```
+
+### POST Request
+```m
+let
+    Body = Json.FromValue([
+        startDate = Date.ToText(param_StartDate, "yyyy-MM-dd"),
+        filters = {[field = "status", value = "active"]}
+    ]),
+    Source = Json.Document(
+        Web.Contents(
+            "https://api.example.com/query",
+            [
+                Headers = [
+                    #"Authorization" = "Bearer " & param_APIToken,
+                    #"Content-Type" = "application/json"
+                ],
+                Content = Body
+            ]
+        )
+    )
+in
+    Source
+```
+
+---
+
+## Excel Files
+
+### Local Excel File
+```m
+let
+    Source = Excel.Workbook(
+        File.Contents(param_FilePath),
+        true,   // Use first row as headers
+        true    // Infer types
+    ),
+    Sheet1 = Source{[Item="Sheet1", Kind="Sheet"]}[Data]
+in
+    Sheet1
+```
+
+### Excel Table (Named Range)
+```m
+let
+    Source = Excel.Workbook(File.Contents(param_FilePath), true, true),
+    SalesTable = Source{[Item="tbl_Sales", Kind="Table"]}[Data]
+in
+    SalesTable
+```
+
+### Current Workbook (Excel Only)
+```m
+let
+    Source = Excel.CurrentWorkbook(){[Name="tbl_Data"]}[Content]
+in
+    Source
+```
+
+---
+
+## CSV Files
+
+### Basic CSV
+```m
+let
+    Source = Csv.Document(
+        File.Contents(param_FilePath),
+        [
+            Delimiter = ",",
+            Encoding = 65001,  // UTF-8
+            QuoteStyle = QuoteStyle.Csv
+        ]
+    ),
+    PromotedHeaders = Table.PromoteHeaders(Source, [PromoteAllScalars=true])
+in
+    PromotedHeaders
+```
+
+### CSV with Specific Columns
+```m
+let
+    Source = Csv.Document(
+        File.Contents(param_FilePath),
+        [
+            Delimiter = ",",
+            Columns = 5,
+            Encoding = 65001,
+            QuoteStyle = QuoteStyle.Csv
+        ]
+    )
+in
+    Source
+```
+
+---
+
+## Folder of Files
+
+### Combine All Files in Folder
+```m
+let
+    Source = Folder.Files(param_FolderPath),
+    // Filter to specific file type
+    FilteredFiles = Table.SelectRows(Source, each Text.EndsWith([Name], ".csv")),
+    // Add file content
+    AddContent = Table.AddColumn(FilteredFiles, "Data", each
+        Csv.Document([Content], [Delimiter=",", Encoding=65001])
+    ),
+    // Expand all rows
+    ExpandedData = Table.ExpandTableColumn(AddContent, "Data",
+        Table.ColumnNames(AddContent{0}[Data])
+    ),
+    // Keep data columns plus filename for reference
+    SelectColumns = Table.SelectColumns(ExpandedData,
+        List.Combine({{"Name"}, Table.ColumnNames(FilteredFiles{0}[Data])})
+    )
+in
+    SelectColumns
+```
+
+### Binary Combine Pattern (Power Query UI)
+```m
+// This is generated by "Combine Files" in Power Query
+let
+    Source = Folder.Files(param_FolderPath),
+    FilteredHiddenFiles = Table.SelectRows(Source, each [Attributes]?[Hidden]? <> true),
+    InvokeCustomFunction = Table.AddColumn(FilteredHiddenFiles, "Transform File", each #"Transform File"([Content])),
+    RemovedOtherColumns = Table.SelectColumns(InvokeCustomFunction, {"Name", "Transform File"}),
+    ExpandedTable = Table.ExpandTableColumn(RemovedOtherColumns, "Transform File", Table.ColumnNames(#"Transform File"(#"Sample File")))
+in
+    ExpandedTable
+```
+
+---
+
+## OData
+
+### Basic OData Connection
+```m
+let
+    Source = OData.Feed("https://services.odata.org/V4/Northwind/Northwind.svc/"),
+    Products = Source{[Name="Products", Signature="table"]}[Data]
+in
+    Products
+```
+
+### With Query Options
+```m
+let
+    Source = OData.Feed(
+        "https://api.example.com/odata/Sales",
+        null,
+        [
+            Query = [
+                #"$filter" = "Year eq 2024",
+                #"$select" = "ID,Date,Amount",
+                #"$orderby" = "Date desc"
+            ]
+        ]
+    )
+in
+    Source
+```
+
+---
+
+## Azure
+
+### Azure SQL Database
+```m
+let
+    Source = Sql.Database(
+        "your-server.database.windows.net",
+        "your-database",
+        [
+            Query = "SELECT * FROM dbo.Sales WHERE Year = 2024"
+        ]
+    )
+in
+    Source
+```
+
+### Azure Blob Storage
+```m
+let
+    Source = AzureStorage.Blobs("https://youraccount.blob.core.windows.net/container"),
+    FilteredFiles = Table.SelectRows(Source, each Text.EndsWith([Name], ".csv")),
+    FirstFile = FilteredFiles{0}[Content],
+    Data = Csv.Document(FirstFile, [Delimiter=",", Encoding=65001])
+in
+    Data
+```
+
+### Azure Data Lake Gen2
+```m
+let
+    Source = AzureStorage.DataLake("https://youraccount.dfs.core.windows.net/filesystem"),
+    NavigateToFolder = Source{[Folder="path/to/folder"]}[Content],
+    CsvFile = NavigateToFolder{[Name="data.csv"]}[Content],
+    Data = Csv.Document(CsvFile, [Delimiter=","])
+in
+    Data
+```
+
+---
+
+## Error Handling for Connections
+
+### Try-Otherwise Pattern
+```m
+let
+    Source = try Sql.Database(param_ServerName, param_DatabaseName) otherwise null,
+    Result = if Source = null then
+        #table({"Error"}, {{"Connection failed"}})
+    else
+        Source{[Schema="dbo", Item="Sales"]}[Data]
+in
+    Result
+```
+
+### Timeout Handling
+```m
+let
+    Source = Sql.Database(
+        param_ServerName,
+        param_DatabaseName,
+        [
+            CommandTimeout = #duration(0, 0, 5, 0),  // 5 minutes
+            CreateNavigationProperties = false       // Faster load
+        ]
+    )
+in
+    Source
+```
+
+---
+
+## Related Resources
+
+- [Parameters](./parameters.md)
+- [Power Query Skill](../../skills/power-query/SKILL.md)

package/library/snippets/power-query/data-cleaning.md

@@ -0,0 +1,298 @@
+# Data Cleaning Power Query Patterns
+
+Reusable M code for cleaning and preparing data.
+
+## Remove Empty and Null Values
+
+### Remove Null Rows
+```m
+let
+    Source = YourSource,
+    // Remove rows where specific column is null
+    FilteredRows = Table.SelectRows(Source, each [ColumnName] <> null)
+in
+    FilteredRows
+```
+
+### Remove Empty Strings
+```m
+let
+    Source = YourSource,
+    FilteredRows = Table.SelectRows(
+        Source,
+        each [ColumnName] <> null and [ColumnName] <> ""
+    )
+in
+    FilteredRows
+```
+
+### Remove Rows with Any Null
+```m
+let
+    Source = YourSource,
+    ColumnNames = Table.ColumnNames(Source),
+    FilteredRows = Table.SelectRows(
+        Source,
+        each not List.Contains(
+            List.Transform(ColumnNames, (col) => Record.Field(_, col)),
+            null
+        )
+    )
+in
+    FilteredRows
+```
+
+### Replace Null with Default
+```m
+let
+    Source = YourSource,
+    ReplacedNull = Table.ReplaceValue(
+        Source,
+        null,
+        0,
+        Replacer.ReplaceValue,
+        {"NumericColumn"}
+    )
+in
+    ReplacedNull
+```
+
+## Text Cleaning
+
+### Trim and Clean Text
+```m
+let
+    Source = YourSource,
+    Cleaned = Table.TransformColumns(
+        Source,
+        {{"TextColumn", each Text.Trim(Text.Clean(_)), type text}}
+    )
+in
+    Cleaned
+```
+
+### Standardize Case
+```m
+let
+    Source = YourSource,
+    // Proper Case
+    ProperCase = Table.TransformColumns(
+        Source,
+        {{"Name", Text.Proper, type text}}
+    ),
+    // Upper Case
+    UpperCase = Table.TransformColumns(
+        Source,
+        {{"Code", Text.Upper, type text}}
+    ),
+    // Lower Case
+    LowerCase = Table.TransformColumns(
+        Source,
+        {{"Email", Text.Lower, type text}}
+    )
+in
+    ProperCase
+```
+
+### Remove Extra Spaces
+```m
+let
+    Source = YourSource,
+    // Replace multiple spaces with single space
+    CleanSpaces = Table.TransformColumns(
+        Source,
+        {{"TextColumn", each Text.Combine(
+            List.Select(Text.Split(_, " "), each _ <> ""),
+            " "
+        ), type text}}
+    )
+in
+    CleanSpaces
+```
+
+### Remove Special Characters
+```m
+let
+    Source = YourSource,
+    RemoveSpecial = Table.TransformColumns(
+        Source,
+        {{"TextColumn", each Text.Select(_, {"A".."Z", "a".."z", "0".."9", " "}), type text}}
+    )
+in
+    RemoveSpecial
+```
+
+## Duplicate Handling
+
+### Remove Duplicate Rows
+```m
+let
+    Source = YourSource,
+    RemovedDuplicates = Table.Distinct(Source)
+in
+    RemovedDuplicates
+```
+
+### Remove Duplicates by Key Column
+```m
+let
+    Source = YourSource,
+    RemovedDuplicates = Table.Distinct(Source, {"KeyColumn"})
+in
+    RemovedDuplicates
+```
+
+### Keep First/Last Occurrence
+```m
+let
+    Source = YourSource,
+    Sorted = Table.Sort(Source, {{"Date", Order.Descending}}),
+    // Keeps first (most recent) per group
+    KeepFirst = Table.Distinct(Sorted, {"CustomerID"})
+in
+    KeepFirst
+```
+
+### Flag Duplicates
+```m
+let
+    Source = YourSource,
+    Grouped = Table.Group(
+        Source,
+        {"KeyColumn"},
+        {{"Count", each Table.RowCount(_), Int64.Type}}
+    ),
+    Merged = Table.NestedJoin(Source, {"KeyColumn"}, Grouped, {"KeyColumn"}, "Counts", JoinKind.LeftOuter),
+    Expanded = Table.ExpandTableColumn(Merged, "Counts", {"Count"}),
+    FlagDuplicates = Table.AddColumn(Expanded, "IsDuplicate", each [Count] > 1, type logical)
+in
+    FlagDuplicates
+```
+
+## Data Type Fixes
+
+### Safe Number Conversion
+```m
+let
+    Source = YourSource,
+    SafeNumber = Table.TransformColumns(
+        Source,
+        {{"Amount", each try Number.From(_) otherwise null, type number}}
+    )
+in
+    SafeNumber
+```
+
+### Safe Date Conversion
+```m
+let
+    Source = YourSource,
+    SafeDate = Table.TransformColumns(
+        Source,
+        {{"DateColumn", each try Date.From(_) otherwise null, type date}}
+    )
+in
+    SafeDate
+```
+
+### Parse Multiple Date Formats
+```m
+let
+    Source = YourSource,
+    ParseDate = Table.TransformColumns(
+        Source,
+        {{"DateColumn", each
+            let
+                TryMDY = try Date.FromText(_, [Format="M/d/yyyy"]),
+                TryDMY = try Date.FromText(_, [Format="d/M/yyyy"]),
+                TryISO = try Date.FromText(_, [Format="yyyy-MM-dd"])
+            in
+                if TryMDY[HasError] = false then TryMDY[Value]
+                else if TryDMY[HasError] = false then TryDMY[Value]
+                else if TryISO[HasError] = false then TryISO[Value]
+                else null
+        , type date}}
+    )
+in
+    ParseDate
+```
+
+## Standardization
+
+### Standardize Column Names
+```m
+let
+    Source = YourSource,
+    // Replace spaces with underscores, remove special chars
+    RenameColumns = Table.TransformColumnNames(
+        Source,
+        each Text.Replace(
+            Text.Replace(
+                Text.Replace(Text.Proper(_), " ", "_"),
+                "#", ""
+            ),
+            "(", ""
+        )
+    )
+in
+    RenameColumns
+```
+
+### Map Values to Standard Names
+```m
+let
+    Source = YourSource,
+    MappingTable = #table(
+        {"Original", "Standard"},
+        {
+            {"NY", "New York"},
+            {"NYC", "New York"},
+            {"CA", "California"},
+            {"Cali", "California"}
+        }
+    ),
+    Merged = Table.NestedJoin(Source, {"State"}, MappingTable, {"Original"}, "Mapping", JoinKind.LeftOuter),
+    Expanded = Table.ExpandTableColumn(Merged, "Mapping", {"Standard"}),
+    Final = Table.AddColumn(Expanded, "CleanState", each if [Standard] = null then [State] else [Standard])
+in
+    Final
+```
+
+## Error Handling
+
+### Replace Errors with Null
+```m
+let
+    Source = YourSource,
+    ReplacedErrors = Table.ReplaceErrorValues(
+        Source,
+        {{"Column1", null}, {"Column2", null}}
+    )
+in
+    ReplacedErrors
+```
+
+### Log Errors to Separate Table
+```m
+let
+    Source = YourSource,
+    AddErrorFlag = Table.AddColumn(
+        Source,
+        "HasError",
+        each try [Amount] > 0 otherwise true,
+        type logical
+    ),
+    ErrorRows = Table.SelectRows(AddErrorFlag, each [HasError] = true),
+    CleanRows = Table.SelectRows(AddErrorFlag, each [HasError] = false)
+in
+    CleanRows
+    // ErrorRows can be returned separately for logging
+```
+
+## Usage Notes
+
+- Always preview results after each transformation
+- Use `try...otherwise` for safe operations
+- Consider query folding when working with databases
+- Document complex transformations with comments
+- Test with edge cases (empty strings, nulls, special characters)