@luquimbo/bi-superpowers 3.1.1 → 3.2.0

package/library/snippets/power-query/connections.md

@@ -1,434 +0,0 @@
-# Data Source Connections
-
-## Overview
-Power Query patterns for connecting to various data sources including databases, cloud services, APIs, and files.
-
----
-
-## SQL Server
-
-### Basic Connection
-```m
-let
-    Source = Sql.Database("server-name", "database-name")
-in
-    Source
-```
-
-### With Native Query
-```m
-let
-    Source = Sql.Database("server-name", "database-name", [
-        Query = "
-            SELECT
-                CustomerID,
-                CustomerName,
-                Email,
-                CreatedDate
-            FROM dbo.Customers
-            WHERE IsActive = 1
-        "
-    ])
-in
-    Source
-```
-
-### With Parameters
-```m
-let
-    Source = Sql.Database(param_ServerName, param_DatabaseName, [
-        Query = "
-            SELECT * FROM dbo.Sales
-            WHERE OrderDate >= '" & Date.ToText(param_StartDate, "yyyy-MM-dd") & "'
-        ",
-        CommandTimeout = #duration(0, 0, 10, 0)  // 10 minutes
-    ])
-in
-    Source
-```
-
-### Navigation to Table
-```m
-let
-    Source = Sql.Database("server-name", "database-name"),
-    Sales = Source{[Schema="dbo", Item="Sales"]}[Data]
-in
-    Sales
-```
-
----
-
-## SharePoint
-
-### SharePoint List
-```m
-let
-    Source = SharePoint.Tables(
-        "https://company.sharepoint.com/sites/SiteName",
-        [Implementation = "2.0", ViewMode = "All"]
-    ),
-    ListData = Source{[Title="ListName"]}[Items],
-    // Remove SharePoint metadata columns
-    RemovedColumns = Table.SelectColumns(ListData, {"Column1", "Column2", "Column3"})
-in
-    RemovedColumns
-```
-
-### SharePoint Folder (Files)
-```m
-let
-    Source = SharePoint.Files(
-        "https://company.sharepoint.com/sites/SiteName",
-        [ApiVersion = 15]
-    ),
-    // Filter to specific folder
-    FilteredFolder = Table.SelectRows(Source, each
-        Text.Contains([Folder Path], "/Documents/Reports/")
-    ),
-    // Filter to Excel files
-    FilteredFiles = Table.SelectRows(FilteredFolder, each
-        Text.EndsWith([Name], ".xlsx")
-    )
-in
-    FilteredFiles
-```
-
-### SharePoint Excel File
-```m
-let
-    Source = SharePoint.Files(
-        "https://company.sharepoint.com/sites/SiteName",
-        [ApiVersion = 15]
-    ),
-    // Find specific file
-    FileRow = Table.SelectRows(Source, each [Name] = "SalesData.xlsx"){0},
-    // Load Excel content
-    ExcelContent = Excel.Workbook(FileRow[Content], true, true),
-    // Get specific sheet
-    Sheet1 = ExcelContent{[Item="Sheet1", Kind="Sheet"]}[Data]
-in
-    Sheet1
-```
-
----
-
-## REST API / Web Services
-
-### Basic JSON API
-```m
-let
-    Source = Json.Document(
-        Web.Contents("https://api.example.com/data")
-    ),
-    ToTable = Table.FromRecords(Source)
-in
-    ToTable
-```
-
-### With Authentication Header
-```m
-let
-    Source = Json.Document(
-        Web.Contents(
-            "https://api.example.com/data",
-            [
-                Headers = [
-                    #"Authorization" = "Bearer " & param_APIToken,
-                    #"Content-Type" = "application/json"
-                ]
-            ]
-        )
-    )
-in
-    Source
-```
-
-### With Query Parameters
-```m
-let
-    Source = Json.Document(
-        Web.Contents(
-            "https://api.example.com/data",
-            [
-                Query = [
-                    startDate = Date.ToText(param_StartDate, "yyyy-MM-dd"),
-                    endDate = Date.ToText(param_EndDate, "yyyy-MM-dd"),
-                    limit = "1000",
-                    page = "1"
-                ]
-            ]
-        )
-    )
-in
-    Source
-```
-
-### Pagination Pattern
-```m
-let
-    // Function to get one page
-    GetPage = (pageNum as number) as table =>
-        let
-            Source = Json.Document(
-                Web.Contents(
-                    "https://api.example.com/data",
-                    [Query = [page = Text.From(pageNum), limit = "100"]]
-                )
-            ),
-            ToTable = Table.FromRecords(Source[data])
-        in
-            ToTable,
-
-    // Get all pages
-    PageCount = 10,  // Or dynamically determine
-    Pages = List.Generate(
-        () => [Page = 1, Data = GetPage(1)],
-        each [Page] <= PageCount,
-        each [Page = [Page] + 1, Data = GetPage([Page] + 1)],
-        each [Data]
-    ),
-    Combined = Table.Combine(Pages)
-in
-    Combined
-```
-
-### POST Request
-```m
-let
-    Body = Json.FromValue([
-        startDate = Date.ToText(param_StartDate, "yyyy-MM-dd"),
-        filters = {[field = "status", value = "active"]}
-    ]),
-    Source = Json.Document(
-        Web.Contents(
-            "https://api.example.com/query",
-            [
-                Headers = [
-                    #"Authorization" = "Bearer " & param_APIToken,
-                    #"Content-Type" = "application/json"
-                ],
-                Content = Body
-            ]
-        )
-    )
-in
-    Source
-```
-
----
-
-## Excel Files
-
-### Local Excel File
-```m
-let
-    Source = Excel.Workbook(
-        File.Contents(param_FilePath),
-        true,   // Use first row as headers
-        true    // Infer types
-    ),
-    Sheet1 = Source{[Item="Sheet1", Kind="Sheet"]}[Data]
-in
-    Sheet1
-```
-
-### Excel Table (Named Range)
-```m
-let
-    Source = Excel.Workbook(File.Contents(param_FilePath), true, true),
-    SalesTable = Source{[Item="tbl_Sales", Kind="Table"]}[Data]
-in
-    SalesTable
-```
-
-### Current Workbook (Excel Only)
-```m
-let
-    Source = Excel.CurrentWorkbook(){[Name="tbl_Data"]}[Content]
-in
-    Source
-```
-
----
-
-## CSV Files
-
-### Basic CSV
-```m
-let
-    Source = Csv.Document(
-        File.Contents(param_FilePath),
-        [
-            Delimiter = ",",
-            Encoding = 65001,  // UTF-8
-            QuoteStyle = QuoteStyle.Csv
-        ]
-    ),
-    PromotedHeaders = Table.PromoteHeaders(Source, [PromoteAllScalars=true])
-in
-    PromotedHeaders
-```
-
-### CSV with Specific Columns
-```m
-let
-    Source = Csv.Document(
-        File.Contents(param_FilePath),
-        [
-            Delimiter = ",",
-            Columns = 5,
-            Encoding = 65001,
-            QuoteStyle = QuoteStyle.Csv
-        ]
-    )
-in
-    Source
-```
-
----
-
-## Folder of Files
-
-### Combine All Files in Folder
-```m
-let
-    Source = Folder.Files(param_FolderPath),
-    // Filter to specific file type
-    FilteredFiles = Table.SelectRows(Source, each Text.EndsWith([Name], ".csv")),
-    // Add file content
-    AddContent = Table.AddColumn(FilteredFiles, "Data", each
-        Csv.Document([Content], [Delimiter=",", Encoding=65001])
-    ),
-    // Expand all rows
-    ExpandedData = Table.ExpandTableColumn(AddContent, "Data",
-        Table.ColumnNames(AddContent{0}[Data])
-    ),
-    // Keep data columns plus filename for reference
-    SelectColumns = Table.SelectColumns(ExpandedData,
-        List.Combine({{"Name"}, Table.ColumnNames(FilteredFiles{0}[Data])})
-    )
-in
-    SelectColumns
-```
-
-### Binary Combine Pattern (Power Query UI)
-```m
-// This is generated by "Combine Files" in Power Query
-let
-    Source = Folder.Files(param_FolderPath),
-    FilteredHiddenFiles = Table.SelectRows(Source, each [Attributes]?[Hidden]? <> true),
-    InvokeCustomFunction = Table.AddColumn(FilteredHiddenFiles, "Transform File", each #"Transform File"([Content])),
-    RemovedOtherColumns = Table.SelectColumns(InvokeCustomFunction, {"Name", "Transform File"}),
-    ExpandedTable = Table.ExpandTableColumn(RemovedOtherColumns, "Transform File", Table.ColumnNames(#"Transform File"(#"Sample File")))
-in
-    ExpandedTable
-```
-
----
-
-## OData
-
-### Basic OData Connection
-```m
-let
-    Source = OData.Feed("https://services.odata.org/V4/Northwind/Northwind.svc/"),
-    Products = Source{[Name="Products", Signature="table"]}[Data]
-in
-    Products
-```
-
-### With Query Options
-```m
-let
-    Source = OData.Feed(
-        "https://api.example.com/odata/Sales",
-        null,
-        [
-            Query = [
-                #"$filter" = "Year eq 2024",
-                #"$select" = "ID,Date,Amount",
-                #"$orderby" = "Date desc"
-            ]
-        ]
-    )
-in
-    Source
-```
-
----
-
-## Azure
-
-### Azure SQL Database
-```m
-let
-    Source = Sql.Database(
-        "your-server.database.windows.net",
-        "your-database",
-        [
-            Query = "SELECT * FROM dbo.Sales WHERE Year = 2024"
-        ]
-    )
-in
-    Source
-```
-
-### Azure Blob Storage
-```m
-let
-    Source = AzureStorage.Blobs("https://youraccount.blob.core.windows.net/container"),
-    FilteredFiles = Table.SelectRows(Source, each Text.EndsWith([Name], ".csv")),
-    FirstFile = FilteredFiles{0}[Content],
-    Data = Csv.Document(FirstFile, [Delimiter=",", Encoding=65001])
-in
-    Data
-```
-
-### Azure Data Lake Gen2
-```m
-let
-    Source = AzureStorage.DataLake("https://youraccount.dfs.core.windows.net/filesystem"),
-    NavigateToFolder = Source{[Folder="path/to/folder"]}[Content],
-    CsvFile = NavigateToFolder{[Name="data.csv"]}[Content],
-    Data = Csv.Document(CsvFile, [Delimiter=","])
-in
-    Data
-```
-
----
-
-## Error Handling for Connections
-
-### Try-Otherwise Pattern
-```m
-let
-    Source = try Sql.Database(param_ServerName, param_DatabaseName) otherwise null,
-    Result = if Source = null then
-        #table({"Error"}, {{"Connection failed"}})
-    else
-        Source{[Schema="dbo", Item="Sales"]}[Data]
-in
-    Result
-```
-
-### Timeout Handling
-```m
-let
-    Source = Sql.Database(
-        param_ServerName,
-        param_DatabaseName,
-        [
-            CommandTimeout = #duration(0, 0, 5, 0),  // 5 minutes
-            CreateNavigationProperties = false       // Faster load
-        ]
-    )
-in
-    Source
-```
-
----
-
-## Related Resources
-
-- [Parameters](./parameters.md)
-- [Power Query Skill](../../skills/power-query/SKILL.md)

package/library/snippets/power-query/data-cleaning.md

@@ -1,298 +0,0 @@
-# Data Cleaning Power Query Patterns
-
-Reusable M code for cleaning and preparing data.
-
-## Remove Empty and Null Values
-
-### Remove Null Rows
-```m
-let
-    Source = YourSource,
-    // Remove rows where specific column is null
-    FilteredRows = Table.SelectRows(Source, each [ColumnName] <> null)
-in
-    FilteredRows
-```
-
-### Remove Empty Strings
-```m
-let
-    Source = YourSource,
-    FilteredRows = Table.SelectRows(
-        Source,
-        each [ColumnName] <> null and [ColumnName] <> ""
-    )
-in
-    FilteredRows
-```
-
-### Remove Rows with Any Null
-```m
-let
-    Source = YourSource,
-    ColumnNames = Table.ColumnNames(Source),
-    FilteredRows = Table.SelectRows(
-        Source,
-        each not List.Contains(
-            List.Transform(ColumnNames, (col) => Record.Field(_, col)),
-            null
-        )
-    )
-in
-    FilteredRows
-```
-
-### Replace Null with Default
-```m
-let
-    Source = YourSource,
-    ReplacedNull = Table.ReplaceValue(
-        Source,
-        null,
-        0,
-        Replacer.ReplaceValue,
-        {"NumericColumn"}
-    )
-in
-    ReplacedNull
-```
-
-## Text Cleaning
-
-### Trim and Clean Text
-```m
-let
-    Source = YourSource,
-    Cleaned = Table.TransformColumns(
-        Source,
-        {{"TextColumn", each Text.Trim(Text.Clean(_)), type text}}
-    )
-in
-    Cleaned
-```
-
-### Standardize Case
-```m
-let
-    Source = YourSource,
-    // Proper Case
-    ProperCase = Table.TransformColumns(
-        Source,
-        {{"Name", Text.Proper, type text}}
-    ),
-    // Upper Case
-    UpperCase = Table.TransformColumns(
-        Source,
-        {{"Code", Text.Upper, type text}}
-    ),
-    // Lower Case
-    LowerCase = Table.TransformColumns(
-        Source,
-        {{"Email", Text.Lower, type text}}
-    )
-in
-    ProperCase
-```
-
-### Remove Extra Spaces
-```m
-let
-    Source = YourSource,
-    // Replace multiple spaces with single space
-    CleanSpaces = Table.TransformColumns(
-        Source,
-        {{"TextColumn", each Text.Combine(
-            List.Select(Text.Split(_, " "), each _ <> ""),
-            " "
-        ), type text}}
-    )
-in
-    CleanSpaces
-```
-
-### Remove Special Characters
-```m
-let
-    Source = YourSource,
-    RemoveSpecial = Table.TransformColumns(
-        Source,
-        {{"TextColumn", each Text.Select(_, {"A".."Z", "a".."z", "0".."9", " "}), type text}}
-    )
-in
-    RemoveSpecial
-```
-
-## Duplicate Handling
-
-### Remove Duplicate Rows
-```m
-let
-    Source = YourSource,
-    RemovedDuplicates = Table.Distinct(Source)
-in
-    RemovedDuplicates
-```
-
-### Remove Duplicates by Key Column
-```m
-let
-    Source = YourSource,
-    RemovedDuplicates = Table.Distinct(Source, {"KeyColumn"})
-in
-    RemovedDuplicates
-```
-
-### Keep First/Last Occurrence
-```m
-let
-    Source = YourSource,
-    Sorted = Table.Sort(Source, {{"Date", Order.Descending}}),
-    // Keeps first (most recent) per group
-    KeepFirst = Table.Distinct(Sorted, {"CustomerID"})
-in
-    KeepFirst
-```
-
-### Flag Duplicates
-```m
-let
-    Source = YourSource,
-    Grouped = Table.Group(
-        Source,
-        {"KeyColumn"},
-        {{"Count", each Table.RowCount(_), Int64.Type}}
-    ),
-    Merged = Table.NestedJoin(Source, {"KeyColumn"}, Grouped, {"KeyColumn"}, "Counts", JoinKind.LeftOuter),
-    Expanded = Table.ExpandTableColumn(Merged, "Counts", {"Count"}),
-    FlagDuplicates = Table.AddColumn(Expanded, "IsDuplicate", each [Count] > 1, type logical)
-in
-    FlagDuplicates
-```
-
-## Data Type Fixes
-
-### Safe Number Conversion
-```m
-let
-    Source = YourSource,
-    SafeNumber = Table.TransformColumns(
-        Source,
-        {{"Amount", each try Number.From(_) otherwise null, type number}}
-    )
-in
-    SafeNumber
-```
-
-### Safe Date Conversion
-```m
-let
-    Source = YourSource,
-    SafeDate = Table.TransformColumns(
-        Source,
-        {{"DateColumn", each try Date.From(_) otherwise null, type date}}
-    )
-in
-    SafeDate
-```
-
-### Parse Multiple Date Formats
-```m
-let
-    Source = YourSource,
-    ParseDate = Table.TransformColumns(
-        Source,
-        {{"DateColumn", each
-            let
-                TryMDY = try Date.FromText(_, [Format="M/d/yyyy"]),
-                TryDMY = try Date.FromText(_, [Format="d/M/yyyy"]),
-                TryISO = try Date.FromText(_, [Format="yyyy-MM-dd"])
-            in
-                if TryMDY[HasError] = false then TryMDY[Value]
-                else if TryDMY[HasError] = false then TryDMY[Value]
-                else if TryISO[HasError] = false then TryISO[Value]
-                else null
-        , type date}}
-    )
-in
-    ParseDate
-```
-
-## Standardization
-
-### Standardize Column Names
-```m
-let
-    Source = YourSource,
-    // Replace spaces with underscores, remove special chars
-    RenameColumns = Table.TransformColumnNames(
-        Source,
-        each Text.Replace(
-            Text.Replace(
-                Text.Replace(Text.Proper(_), " ", "_"),
-                "#", ""
-            ),
-            "(", ""
-        )
-    )
-in
-    RenameColumns
-```
-
-### Map Values to Standard Names
-```m
-let
-    Source = YourSource,
-    MappingTable = #table(
-        {"Original", "Standard"},
-        {
-            {"NY", "New York"},
-            {"NYC", "New York"},
-            {"CA", "California"},
-            {"Cali", "California"}
-        }
-    ),
-    Merged = Table.NestedJoin(Source, {"State"}, MappingTable, {"Original"}, "Mapping", JoinKind.LeftOuter),
-    Expanded = Table.ExpandTableColumn(Merged, "Mapping", {"Standard"}),
-    Final = Table.AddColumn(Expanded, "CleanState", each if [Standard] = null then [State] else [Standard])
-in
-    Final
-```
-
-## Error Handling
-
-### Replace Errors with Null
-```m
-let
-    Source = YourSource,
-    ReplacedErrors = Table.ReplaceErrorValues(
-        Source,
-        {{"Column1", null}, {"Column2", null}}
-    )
-in
-    ReplacedErrors
-```
-
-### Log Errors to Separate Table
-```m
-let
-    Source = YourSource,
-    AddErrorFlag = Table.AddColumn(
-        Source,
-        "HasError",
-        each try [Amount] > 0 otherwise true,
-        type logical
-    ),
-    ErrorRows = Table.SelectRows(AddErrorFlag, each [HasError] = true),
-    CleanRows = Table.SelectRows(AddErrorFlag, each [HasError] = false)
-in
-    CleanRows
-    // ErrorRows can be returned separately for logging
-```
-
-## Usage Notes
-
-- Always preview results after each transformation
-- Use `try...otherwise` for safe operations
-- Consider query folding when working with databases
-- Document complex transformations with comments
-- Test with edge cases (empty strings, nulls, special characters)