@luquimbo/bi-superpowers 3.1.1 → 4.1.0

package/library/snippets/power-query/error-handling.md

@@ -1,526 +0,0 @@
-# Error Handling in Power Query
-
-Patterns for handling errors, debugging, and building robust data pipelines.
-
----
-
-## Try-Otherwise Pattern
-
-### Basic Error Handling
-
-```m
-let
-    Source = YourSource,
-
-    // Safe transformation - returns null on error
-    SafeConvert = Table.TransformColumns(Source, {
-        {"Amount", each try Number.From(_) otherwise null, type number}
-    })
-in
-    SafeConvert
-```
-
-### With Custom Default Value
-
-```m
-let
-    Source = YourSource,
-
-    // Return 0 instead of null on error
-    SafeAmount = Table.TransformColumns(Source, {
-        {"Amount", each try Number.From(_) otherwise 0, type number}
-    })
-in
-    SafeAmount
-```
-
-### With Error Logging
-
-```m
-let
-    Source = YourSource,
-
-    // Capture both value and error status
-    WithErrorInfo = Table.AddColumn(Source, "ParseResult",
-        each
-            let
-                Result = try Number.From([Amount])
-            in
-                [
-                    Value = if Result[HasError] then null else Result[Value],
-                    HasError = Result[HasError],
-                    Error = if Result[HasError] then Result[Error][Message] else null
-                ]
-    ),
-
-    // Expand the record
-    Expanded = Table.ExpandRecordColumn(WithErrorInfo, "ParseResult",
-        {"Value", "HasError", "Error"},
-        {"Amount_Parsed", "Amount_HasError", "Amount_Error"}
-    )
-in
-    Expanded
-```
-
----
-
-## Try Expression Details
-
-### Structure of Try Result
-
-```m
-let
-    // try returns a record with these fields:
-    // [HasError] - true/false
-    // [Value] - result if no error
-    // [Error] - error record if error occurred
-
-    Result = try Number.From("abc"),
-
-    // Result = [HasError = true, Error = [Reason = "...", Message = "...", Detail = "..."]]
-
-    SafeValue = if Result[HasError] then 0 else Result[Value]
-in
-    SafeValue
-```
-
-### Access Error Details
-
-```m
-let
-    Source = YourSource,
-
-    WithErrorDetails = Table.AddColumn(Source, "ErrorInfo",
-        each
-            let
-                Result = try SomeRiskyOperation([ColumnA])
-            in
-                if Result[HasError] then
-                    "Error: " & Result[Error][Reason] &
-                    " - " & Result[Error][Message]
-                else
-                    "Success"
-    )
-in
-    WithErrorDetails
-```
-
----
-
-## Null Handling
-
-### COALESCE Pattern (First Non-Null)
-
-```m
-let
-    Source = YourSource,
-
-    // Return first non-null value
-    WithCoalesce = Table.AddColumn(Source, "ContactEmail",
-        each
-            if [PrimaryEmail] <> null then [PrimaryEmail]
-            else if [SecondaryEmail] <> null then [SecondaryEmail]
-            else if [WorkEmail] <> null then [WorkEmail]
-            else "unknown@example.com",
-        type text
-    )
-in
-    WithCoalesce
-```
-
-### Null-Safe Operations
-
-```m
-let
-    Source = YourSource,
-
-    // Safe text operations (handle null)
-    SafeText = Table.TransformColumns(Source, {
-        {"Name", each if _ = null then "" else Text.Trim(_), type text}
-    }),
-
-    // Safe number operations
-    SafeNumber = Table.TransformColumns(Source, {
-        {"Amount", each if _ = null then 0 else _, type number}
-    })
-in
-    SafeNumber
-```
-
-### Check for Null Before Operation
-
-```m
-let
-    Source = YourSource,
-
-    SafeCalculation = Table.AddColumn(Source, "Result",
-        each
-            if [Denominator] = null or [Denominator] = 0 then
-                null
-            else
-                [Numerator] / [Denominator],
-        type number
-    )
-in
-    SafeCalculation
-```
-
----
-
-## Error Row Separation
-
-### Split Clean and Error Rows
-
-```m
-let
-    Source = YourSource,
-
-    // Add error flag
-    WithFlag = Table.AddColumn(Source, "_HasError",
-        each
-            try (
-                // Validate required fields
-                [CustomerID] <> null and
-                [Amount] <> null and
-                Number.From([Amount]) >= 0
-            ) otherwise false,
-        type logical
-    ),
-
-    // Separate good and bad rows
-    CleanRows = Table.SelectRows(WithFlag, each [_HasError] = true),
-    ErrorRows = Table.SelectRows(WithFlag, each [_HasError] = false),
-
-    // Remove flag from clean data
-    CleanData = Table.RemoveColumns(CleanRows, {"_HasError"})
-in
-    CleanData  // or return both via Record
-```
-
-### Return Both Clean and Error Tables
-
-```m
-let
-    Source = YourSource,
-
-    // Validation function
-    ValidateRow = (row as record) as logical =>
-        row[CustomerID] <> null and
-        row[Amount] <> null and
-        (try Number.From(row[Amount]) >= 0 otherwise false),
-
-    // Flag rows
-    Flagged = Table.AddColumn(Source, "_IsValid", each ValidateRow(_), type logical),
-
-    // Split
-    Result = [
-        CleanData = Table.SelectRows(Flagged, each [_IsValid] = true),
-        ErrorData = Table.SelectRows(Flagged, each [_IsValid] = false),
-        ErrorCount = Table.RowCount(Table.SelectRows(Flagged, each [_IsValid] = false)),
-        CleanCount = Table.RowCount(Table.SelectRows(Flagged, each [_IsValid] = true))
-    ]
-in
-    Result
-```
-
----
-
-## Replace Errors
-
-### Replace All Errors in Table
-
-```m
-let
-    Source = YourSource,
-
-    // Replace errors in specific columns
-    ReplacedErrors = Table.ReplaceErrorValues(Source, {
-        {"Amount", 0},
-        {"Quantity", 0},
-        {"Date", null},
-        {"Name", "Unknown"}
-    })
-in
-    ReplacedErrors
-```
-
-### Replace Errors in All Columns
-
-```m
-let
-    Source = YourSource,
-    ColumnNames = Table.ColumnNames(Source),
-
-    // Create replacement list for all columns
-    ReplacementList = List.Transform(
-        ColumnNames,
-        each {_, null}  // Replace with null
-    ),
-
-    ReplacedErrors = Table.ReplaceErrorValues(Source, ReplacementList)
-in
-    ReplacedErrors
-```
-
----
-
-## Validation Patterns
-
-### Data Type Validation
-
-```m
-let
-    Source = YourSource,
-
-    // Validate data types before processing
-    Validated = Table.AddColumn(Source, "ValidationResult",
-        each
-            let
-                Checks = {
-                    {[CustomerID] <> null, "CustomerID is required"},
-                    {try Number.From([Amount]) <> null otherwise false, "Amount must be numeric"},
-                    {try Date.From([OrderDate]) <> null otherwise false, "OrderDate must be valid date"},
-                    {[Status] <> null and List.Contains({"Open", "Closed", "Pending"}, [Status]), "Invalid Status"}
-                },
-                Failures = List.Select(Checks, each not _{0}),
-                Messages = List.Transform(Failures, each _{1})
-            in
-                if List.Count(Messages) = 0 then "Valid"
-                else Text.Combine(Messages, "; ")
-    )
-in
-    Validated
-```
-
-### Business Rule Validation
-
-```m
-let
-    Source = YourSource,
-
-    ValidatedData = Table.AddColumn(Source, "BusinessRuleCheck",
-        each
-            let
-                Rules = {
-                    // Rule 1: Amount must be positive
-                    {[Amount] > 0, "Amount must be positive"},
-
-                    // Rule 2: Discount cannot exceed amount
-                    {[Discount] <= [Amount], "Discount exceeds amount"},
-
-                    // Rule 3: Order date must be in the past
-                    {[OrderDate] <= DateTime.Date(DateTime.LocalNow()), "Future order date"},
-
-                    // Rule 4: Customer must exist
-                    {[CustomerID] <> null, "Missing customer"}
-                },
-                Failures = List.Select(Rules, each not _{0}),
-                ErrorMessages = List.Transform(Failures, each _{1})
-            in
-                [
-                    IsValid = List.Count(ErrorMessages) = 0,
-                    Errors = Text.Combine(ErrorMessages, "; ")
-                ]
-    ),
-
-    // Expand validation results
-    Expanded = Table.ExpandRecordColumn(ValidatedData, "BusinessRuleCheck",
-        {"IsValid", "Errors"}
-    )
-in
-    Expanded
-```
-
----
-
-## Connection Error Handling
-
-### Handle Missing Data Source
-
-```m
-let
-    // Try to connect, fall back gracefully
-    TryConnection = try Sql.Database("server", "database"),
-
-    Result =
-        if TryConnection[HasError] then
-            #table(
-                {"Status", "Message"},
-                {{"Error", "Could not connect to database: " & TryConnection[Error][Message]}}
-            )
-        else
-            let
-                Source = TryConnection[Value],
-                Data = Source{[Schema="dbo", Item="Sales"]}[Data]
-            in
-                Data
-in
-    Result
-```
-
-### File Existence Check
-
-```m
-let
-    FilePath = "C:\Data\input.csv",
-
-    // Check if file exists
-    FileExists = try File.Contents(FilePath) <> null otherwise false,
-
-    Result =
-        if not FileExists then
-            error Error.Record("File Not Found", "The file " & FilePath & " does not exist")
-        else
-            Csv.Document(File.Contents(FilePath), [Delimiter=","])
-in
-    Result
-```
-
----
-
-## Debugging Helpers
-
-### Add Row Numbers for Tracking
-
-```m
-let
-    Source = YourSource,
-
-    // Add index for debugging
-    WithIndex = Table.AddIndexColumn(Source, "_RowNumber", 1, 1, Int64.Type),
-
-    // Now you can reference specific rows in error messages
-    Processed = Table.TransformColumns(WithIndex, {
-        {"Amount", each try Number.From(_) otherwise
-            error Error.Record("Parse Error", "Row " & Text.From([_RowNumber]) & ": Invalid amount")}
-    })
-in
-    Processed
-```
-
-### Log Processing Steps
-
-```m
-let
-    // Create log entries
-    Log = (message as text) => Diagnostics.Trace(TraceLevel.Information, message, () => true, true),
-
-    Source = Log("Starting query") then YourSource,
-    Step1 = Log("Filtering rows") then Table.SelectRows(Source, each [Status] = "Active"),
-    Step2 = Log("Transforming columns") then Table.TransformColumns(Step1, {{"Amount", each _ * 1.1}}),
-    Final = Log("Query complete, rows: " & Text.From(Table.RowCount(Step2))) then Step2
-in
-    Final
-```
-
-### Conditional Debug Output
-
-```m
-let
-    DebugMode = true,  // Set to false in production
-
-    Source = YourSource,
-
-    // Only add debug columns in debug mode
-    WithDebug =
-        if DebugMode then
-            Table.AddColumn(
-                Table.AddColumn(Source, "_DebugRowNum", each [Index]),
-                "_DebugValues", each Text.From([Amount]) & "," & [Status]
-            )
-        else
-            Source
-in
-    WithDebug
-```
-
----
-
-## Custom Error Creation
-
-### Raise Custom Error
-
-```m
-let
-    Source = YourSource,
-    RowCount = Table.RowCount(Source),
-
-    // Validate before processing
-    Validated =
-        if RowCount = 0 then
-            error Error.Record(
-                "EmptySourceError",
-                "Source data is empty",
-                [RowCount = 0, Source = "YourSource"]
-            )
-        else if RowCount > 1000000 then
-            error Error.Record(
-                "DataTooLargeError",
-                "Source has too many rows: " & Text.From(RowCount),
-                [RowCount = RowCount, MaxAllowed = 1000000]
-            )
-        else
-            Source
-in
-    Validated
-```
-
-### Error with Details
-
-```m
-let
-    ValidateAmount = (amount as any) =>
-        let
-            NumericValue = try Number.From(amount)
-        in
-            if NumericValue[HasError] then
-                error Error.Record(
-                    "InvalidAmountError",
-                    "Could not parse amount: " & Text.From(amount),
-                    [
-                        OriginalValue = amount,
-                        OriginalType = Value.Type(amount),
-                        ParseError = NumericValue[Error][Message]
-                    ]
-                )
-            else if NumericValue[Value] < 0 then
-                error Error.Record(
-                    "NegativeAmountError",
-                    "Amount cannot be negative",
-                    [Value = NumericValue[Value]]
-                )
-            else
-                NumericValue[Value],
-
-    Source = YourSource,
-    Validated = Table.TransformColumns(Source, {
-        {"Amount", ValidateAmount, type number}
-    })
-in
-    Validated
-```
-
----
-
-## Best Practices Summary
-
-### Error Handling Checklist
-
-| Scenario | Pattern |
-|----------|---------|
-| Type conversion | `try Number.From(_) otherwise null` |
-| Null values | Check with `= null` before operation |
-| Division | Check denominator before dividing |
-| Missing columns | `try Record.Field(_, "Column") otherwise` |
-| Connection errors | Wrap source in `try` block |
-| Validation | Add validation column, filter errors |
-| Debugging | Add row numbers, use Diagnostics.Trace |
-
-### Don'ts
-
-| Bad Practice | Why |
-|--------------|-----|
-| Ignoring errors silently | Hides data quality issues |
-| Catching too broadly | May mask real problems |
-| Not logging errors | Can't diagnose issues |
-| Failing on first error | Loses batch visibility |