@luquimbo/bi-superpowers 1.0.0

package/library/snippets/power-query/error-handling.md

@@ -0,0 +1,526 @@
+# Error Handling in Power Query
+
+Patterns for handling errors, debugging, and building robust data pipelines.
+
+---
+
+## Try-Otherwise Pattern
+
+### Basic Error Handling
+
+```m
+let
+    Source = YourSource,
+
+    // Safe transformation - returns null on error
+    SafeConvert = Table.TransformColumns(Source, {
+        {"Amount", each try Number.From(_) otherwise null, type number}
+    })
+in
+    SafeConvert
+```
+
+### With Custom Default Value
+
+```m
+let
+    Source = YourSource,
+
+    // Return 0 instead of null on error
+    SafeAmount = Table.TransformColumns(Source, {
+        {"Amount", each try Number.From(_) otherwise 0, type number}
+    })
+in
+    SafeAmount
+```
+
+### With Error Logging
+
+```m
+let
+    Source = YourSource,
+
+    // Capture both value and error status
+    WithErrorInfo = Table.AddColumn(Source, "ParseResult",
+        each
+            let
+                Result = try Number.From([Amount])
+            in
+                [
+                    Value = if Result[HasError] then null else Result[Value],
+                    HasError = Result[HasError],
+                    Error = if Result[HasError] then Result[Error][Message] else null
+                ]
+    ),
+
+    // Expand the record
+    Expanded = Table.ExpandRecordColumn(WithErrorInfo, "ParseResult",
+        {"Value", "HasError", "Error"},
+        {"Amount_Parsed", "Amount_HasError", "Amount_Error"}
+    )
+in
+    Expanded
+```
+
+---
+
+## Try Expression Details
+
+### Structure of Try Result
+
+```m
+let
+    // try returns a record with these fields:
+    // [HasError] - true/false
+    // [Value] - result if no error
+    // [Error] - error record if error occurred
+
+    Result = try Number.From("abc"),
+
+    // Result = [HasError = true, Error = [Reason = "...", Message = "...", Detail = "..."]]
+
+    SafeValue = if Result[HasError] then 0 else Result[Value]
+in
+    SafeValue
+```
+
+### Access Error Details
+
+```m
+let
+    Source = YourSource,
+
+    WithErrorDetails = Table.AddColumn(Source, "ErrorInfo",
+        each
+            let
+                Result = try SomeRiskyOperation([ColumnA])
+            in
+                if Result[HasError] then
+                    "Error: " & Result[Error][Reason] &
+                    " - " & Result[Error][Message]
+                else
+                    "Success"
+    )
+in
+    WithErrorDetails
+```
+
+---
+
+## Null Handling
+
+### COALESCE Pattern (First Non-Null)
+
+```m
+let
+    Source = YourSource,
+
+    // Return first non-null value
+    WithCoalesce = Table.AddColumn(Source, "ContactEmail",
+        each
+            if [PrimaryEmail] <> null then [PrimaryEmail]
+            else if [SecondaryEmail] <> null then [SecondaryEmail]
+            else if [WorkEmail] <> null then [WorkEmail]
+            else "unknown@example.com",
+        type text
+    )
+in
+    WithCoalesce
+```
+
+### Null-Safe Operations
+
+```m
+let
+    Source = YourSource,
+
+    // Safe text operations (handle null)
+    SafeText = Table.TransformColumns(Source, {
+        {"Name", each if _ = null then "" else Text.Trim(_), type text}
+    }),
+
+    // Safe number operations
+    SafeNumber = Table.TransformColumns(Source, {
+        {"Amount", each if _ = null then 0 else _, type number}
+    })
+in
+    SafeNumber
+```
+
+### Check for Null Before Operation
+
+```m
+let
+    Source = YourSource,
+
+    SafeCalculation = Table.AddColumn(Source, "Result",
+        each
+            if [Denominator] = null or [Denominator] = 0 then
+                null
+            else
+                [Numerator] / [Denominator],
+        type number
+    )
+in
+    SafeCalculation
+```
+
+---
+
+## Error Row Separation
+
+### Split Clean and Error Rows
+
+```m
+let
+    Source = YourSource,
+
+    // Add error flag
+    WithFlag = Table.AddColumn(Source, "_HasError",
+        each
+            try (
+                // Validate required fields
+                [CustomerID] <> null and
+                [Amount] <> null and
+                Number.From([Amount]) >= 0
+            ) otherwise false,
+        type logical
+    ),
+
+    // Separate good and bad rows
+    CleanRows = Table.SelectRows(WithFlag, each [_HasError] = true),
+    ErrorRows = Table.SelectRows(WithFlag, each [_HasError] = false),
+
+    // Remove flag from clean data
+    CleanData = Table.RemoveColumns(CleanRows, {"_HasError"})
+in
+    CleanData  // or return both via Record
+```
+
+### Return Both Clean and Error Tables
+
+```m
+let
+    Source = YourSource,
+
+    // Validation function
+    ValidateRow = (row as record) as logical =>
+        row[CustomerID] <> null and
+        row[Amount] <> null and
+        (try Number.From(row[Amount]) >= 0 otherwise false),
+
+    // Flag rows
+    Flagged = Table.AddColumn(Source, "_IsValid", each ValidateRow(_), type logical),
+
+    // Split
+    Result = [
+        CleanData = Table.SelectRows(Flagged, each [_IsValid] = true),
+        ErrorData = Table.SelectRows(Flagged, each [_IsValid] = false),
+        ErrorCount = Table.RowCount(Table.SelectRows(Flagged, each [_IsValid] = false)),
+        CleanCount = Table.RowCount(Table.SelectRows(Flagged, each [_IsValid] = true))
+    ]
+in
+    Result
+```
+
+---
+
+## Replace Errors
+
+### Replace All Errors in Table
+
+```m
+let
+    Source = YourSource,
+
+    // Replace errors in specific columns
+    ReplacedErrors = Table.ReplaceErrorValues(Source, {
+        {"Amount", 0},
+        {"Quantity", 0},
+        {"Date", null},
+        {"Name", "Unknown"}
+    })
+in
+    ReplacedErrors
+```
+
+### Replace Errors in All Columns
+
+```m
+let
+    Source = YourSource,
+    ColumnNames = Table.ColumnNames(Source),
+
+    // Create replacement list for all columns
+    ReplacementList = List.Transform(
+        ColumnNames,
+        each {_, null}  // Replace with null
+    ),
+
+    ReplacedErrors = Table.ReplaceErrorValues(Source, ReplacementList)
+in
+    ReplacedErrors
+```
+
+---
+
+## Validation Patterns
+
+### Data Type Validation
+
+```m
+let
+    Source = YourSource,
+
+    // Validate data types before processing
+    Validated = Table.AddColumn(Source, "ValidationResult",
+        each
+            let
+                Checks = {
+                    {[CustomerID] <> null, "CustomerID is required"},
+                    {try Number.From([Amount]) <> null otherwise false, "Amount must be numeric"},
+                    {try Date.From([OrderDate]) <> null otherwise false, "OrderDate must be valid date"},
+                    {[Status] <> null and List.Contains({"Open", "Closed", "Pending"}, [Status]), "Invalid Status"}
+                },
+                Failures = List.Select(Checks, each not _{0}),
+                Messages = List.Transform(Failures, each _{1})
+            in
+                if List.Count(Messages) = 0 then "Valid"
+                else Text.Combine(Messages, "; ")
+    )
+in
+    Validated
+```
+
+### Business Rule Validation
+
+```m
+let
+    Source = YourSource,
+
+    ValidatedData = Table.AddColumn(Source, "BusinessRuleCheck",
+        each
+            let
+                Rules = {
+                    // Rule 1: Amount must be positive
+                    {[Amount] > 0, "Amount must be positive"},
+
+                    // Rule 2: Discount cannot exceed amount
+                    {[Discount] <= [Amount], "Discount exceeds amount"},
+
+                    // Rule 3: Order date must be in the past
+                    {[OrderDate] <= DateTime.Date(DateTime.LocalNow()), "Future order date"},
+
+                    // Rule 4: Customer must exist
+                    {[CustomerID] <> null, "Missing customer"}
+                },
+                Failures = List.Select(Rules, each not _{0}),
+                ErrorMessages = List.Transform(Failures, each _{1})
+            in
+                [
+                    IsValid = List.Count(ErrorMessages) = 0,
+                    Errors = Text.Combine(ErrorMessages, "; ")
+                ]
+    ),
+
+    // Expand validation results
+    Expanded = Table.ExpandRecordColumn(ValidatedData, "BusinessRuleCheck",
+        {"IsValid", "Errors"}
+    )
+in
+    Expanded
+```
+
+---
+
+## Connection Error Handling
+
+### Handle Missing Data Source
+
+```m
+let
+    // Try to connect, fall back gracefully
+    TryConnection = try Sql.Database("server", "database"),
+
+    Result =
+        if TryConnection[HasError] then
+            #table(
+                {"Status", "Message"},
+                {{"Error", "Could not connect to database: " & TryConnection[Error][Message]}}
+            )
+        else
+            let
+                Source = TryConnection[Value],
+                Data = Source{[Schema="dbo", Item="Sales"]}[Data]
+            in
+                Data
+in
+    Result
+```
+
+### File Existence Check
+
+```m
+let
+    FilePath = "C:\Data\input.csv",
+
+    // Check if file exists
+    FileExists = try File.Contents(FilePath) <> null otherwise false,
+
+    Result =
+        if not FileExists then
+            error Error.Record("File Not Found", "The file " & FilePath & " does not exist")
+        else
+            Csv.Document(File.Contents(FilePath), [Delimiter=","])
+in
+    Result
+```
+
+---
+
+## Debugging Helpers
+
+### Add Row Numbers for Tracking
+
+```m
+let
+    Source = YourSource,
+
+    // Add index for debugging
+    WithIndex = Table.AddIndexColumn(Source, "_RowNumber", 1, 1, Int64.Type),
+
+    // Now you can reference specific rows in error messages
+    Processed = Table.TransformColumns(WithIndex, {
+        {"Amount", each try Number.From(_) otherwise
+            error Error.Record("Parse Error", "Row " & Text.From([_RowNumber]) & ": Invalid amount")}
+    })
+in
+    Processed
+```
+
+### Log Processing Steps
+
+```m
+let
+    // Create log entries
+    Log = (message as text) => Diagnostics.Trace(TraceLevel.Information, message, () => true, true),
+
+    Source = Log("Starting query") then YourSource,
+    Step1 = Log("Filtering rows") then Table.SelectRows(Source, each [Status] = "Active"),
+    Step2 = Log("Transforming columns") then Table.TransformColumns(Step1, {{"Amount", each _ * 1.1}}),
+    Final = Log("Query complete, rows: " & Text.From(Table.RowCount(Step2))) then Step2
+in
+    Final
+```
+
+### Conditional Debug Output
+
+```m
+let
+    DebugMode = true,  // Set to false in production
+
+    Source = YourSource,
+
+    // Only add debug columns in debug mode
+    WithDebug =
+        if DebugMode then
+            Table.AddColumn(
+                Table.AddColumn(Source, "_DebugRowNum", each [Index]),
+                "_DebugValues", each Text.From([Amount]) & "," & [Status]
+            )
+        else
+            Source
+in
+    WithDebug
+```
+
+---
+
+## Custom Error Creation
+
+### Raise Custom Error
+
+```m
+let
+    Source = YourSource,
+    RowCount = Table.RowCount(Source),
+
+    // Validate before processing
+    Validated =
+        if RowCount = 0 then
+            error Error.Record(
+                "EmptySourceError",
+                "Source data is empty",
+                [RowCount = 0, Source = "YourSource"]
+            )
+        else if RowCount > 1000000 then
+            error Error.Record(
+                "DataTooLargeError",
+                "Source has too many rows: " & Text.From(RowCount),
+                [RowCount = RowCount, MaxAllowed = 1000000]
+            )
+        else
+            Source
+in
+    Validated
+```
+
+### Error with Details
+
+```m
+let
+    ValidateAmount = (amount as any) =>
+        let
+            NumericValue = try Number.From(amount)
+        in
+            if NumericValue[HasError] then
+                error Error.Record(
+                    "InvalidAmountError",
+                    "Could not parse amount: " & Text.From(amount),
+                    [
+                        OriginalValue = amount,
+                        OriginalType = Value.Type(amount),
+                        ParseError = NumericValue[Error][Message]
+                    ]
+                )
+            else if NumericValue[Value] < 0 then
+                error Error.Record(
+                    "NegativeAmountError",
+                    "Amount cannot be negative",
+                    [Value = NumericValue[Value]]
+                )
+            else
+                NumericValue[Value],
+
+    Source = YourSource,
+    Validated = Table.TransformColumns(Source, {
+        {"Amount", ValidateAmount, type number}
+    })
+in
+    Validated
+```
+
+---
+
+## Best Practices Summary
+
+### Error Handling Checklist
+
+| Scenario | Pattern |
+|----------|---------|
+| Type conversion | `try Number.From(_) otherwise null` |
+| Null values | Check with `= null` before operation |
+| Division | Check denominator before dividing |
+| Missing columns | `try Record.Field(_, "Column") otherwise` |
+| Connection errors | Wrap source in `try` block |
+| Validation | Add validation column, filter errors |
+| Debugging | Add row numbers, use Diagnostics.Trace |
+
+### Don'ts
+
+| Bad Practice | Why |
+|--------------|-----|
+| Ignoring errors silently | Hides data quality issues |
+| Catching too broadly | May mask real problems |
+| Not logging errors | Can't diagnose issues |
+| Failing on first error | Loses batch visibility |