@luquimbo/bi-superpowers 1.0.0

package/library/snippets/power-query/api-integration.md

@@ -0,0 +1,707 @@
+# API Integration in Power Query
+
+Patterns for connecting to REST APIs, handling pagination, authentication, and error handling.
+
+---
+
+## Basic REST API Calls
+
+### Simple GET Request
+
+```m
+let
+    // Basic API call
+    Source = Json.Document(
+        Web.Contents("https://api.example.com/data")
+    ),
+
+    // Convert to table
+    ToTable = Table.FromRecords(Source)
+in
+    ToTable
+```
+
+### GET with Query Parameters
+
+```m
+let
+    BaseUrl = "https://api.example.com/data",
+    QueryParams = [
+        startDate = "2024-01-01",
+        endDate = "2024-12-31",
+        limit = "1000"
+    ],
+
+    Source = Json.Document(
+        Web.Contents(BaseUrl, [Query = QueryParams])
+    )
+in
+    Source
+```
+
+### GET with Headers
+
+```m
+let
+    Source = Json.Document(
+        Web.Contents(
+            "https://api.example.com/data",
+            [
+                Headers = [
+                    #"Content-Type" = "application/json",
+                    #"Accept" = "application/json",
+                    #"X-Custom-Header" = "value"
+                ]
+            ]
+        )
+    )
+in
+    Source
+```
+
+---
+
+## Authentication Patterns
+
+### API Key in Header
+
+```m
+let
+    ApiKey = "your-api-key-here",  // Use parameter in production
+
+    Source = Json.Document(
+        Web.Contents(
+            "https://api.example.com/data",
+            [
+                Headers = [
+                    #"Authorization" = "Bearer " & ApiKey,
+                    #"Content-Type" = "application/json"
+                ]
+            ]
+        )
+    )
+in
+    Source
+```
+
+### API Key in Query String
+
+```m
+let
+    ApiKey = "your-api-key-here",
+
+    Source = Json.Document(
+        Web.Contents(
+            "https://api.example.com/data",
+            [
+                Query = [
+                    api_key = ApiKey,
+                    format = "json"
+                ]
+            ]
+        )
+    )
+in
+    Source
+```
+
+### Basic Authentication
+
+```m
+let
+    Username = "user",
+    Password = "pass",
+
+    // Encode credentials
+    Credentials = Binary.ToText(
+        Text.ToBinary(Username & ":" & Password),
+        BinaryEncoding.Base64
+    ),
+
+    Source = Json.Document(
+        Web.Contents(
+            "https://api.example.com/data",
+            [
+                Headers = [
+                    #"Authorization" = "Basic " & Credentials
+                ]
+            ]
+        )
+    )
+in
+    Source
+```
+
+### OAuth Token (Pre-obtained)
+
+```m
+let
+    // Token obtained separately (e.g., via Azure AD)
+    AccessToken = "eyJhbGciOiJSUzI1NiIs...",
+
+    Source = Json.Document(
+        Web.Contents(
+            "https://api.example.com/data",
+            [
+                Headers = [
+                    #"Authorization" = "Bearer " & AccessToken
+                ]
+            ]
+        )
+    )
+in
+    Source
+```
+
+---
+
+## Pagination Patterns
+
+### Offset-Based Pagination
+
+```m
+let
+    BaseUrl = "https://api.example.com/data",
+    PageSize = 100,
+
+    // Function to get one page
+    GetPage = (offset as number) =>
+        let
+            Response = Json.Document(
+                Web.Contents(
+                    BaseUrl,
+                    [Query = [
+                        offset = Text.From(offset),
+                        limit = Text.From(PageSize)
+                    ]]
+                )
+            )
+        in
+            Response[data],
+
+    // Get first page to check total
+    FirstPage = GetPage(0),
+    TotalCount = Json.Document(
+        Web.Contents(BaseUrl, [Query = [limit = "1"]])
+    )[total],
+
+    // Generate list of offsets
+    Offsets = List.Generate(
+        () => 0,
+        each _ < TotalCount,
+        each _ + PageSize
+    ),
+
+    // Get all pages
+    AllPages = List.Transform(Offsets, each GetPage(_)),
+
+    // Combine into single table
+    Combined = Table.FromList(
+        List.Combine(AllPages),
+        Splitter.SplitByNothing(),
+        {"Column1"}
+    ),
+
+    Expanded = Table.ExpandRecordColumn(Combined, "Column1", Record.FieldNames(AllPages{0}{0}))
+in
+    Expanded
+```
+
+### Page Number Pagination
+
+```m
+let
+    BaseUrl = "https://api.example.com/data",
+
+    // Function to get one page
+    GetPage = (pageNum as number) =>
+        let
+            Response = Json.Document(
+                Web.Contents(
+                    BaseUrl,
+                    [Query = [page = Text.From(pageNum)]]
+                )
+            )
+        in
+            Response,
+
+    // Get first page
+    FirstPage = GetPage(1),
+    TotalPages = FirstPage[totalPages],
+
+    // Generate page numbers
+    PageNumbers = {1..TotalPages},
+
+    // Get all pages
+    AllResponses = List.Transform(PageNumbers, each GetPage(_)),
+
+    // Extract data from each response
+    AllData = List.Transform(AllResponses, each _[data]),
+
+    // Combine
+    Combined = List.Combine(AllData),
+    ToTable = Table.FromRecords(Combined)
+in
+    ToTable
+```
+
+### Cursor-Based Pagination
+
+```m
+let
+    BaseUrl = "https://api.example.com/data",
+
+    // Recursive function for cursor pagination
+    GetAllPages = (cursor as nullable text, accumulator as list) =>
+        let
+            QueryParams = if cursor = null then [] else [cursor = cursor],
+            Response = Json.Document(
+                Web.Contents(BaseUrl, [Query = QueryParams])
+            ),
+            CurrentData = Response[data],
+            NextCursor = try Response[next_cursor] otherwise null,
+            NewAccumulator = accumulator & CurrentData,
+
+            Result = if NextCursor = null then
+                NewAccumulator
+            else
+                @GetAllPages(NextCursor, NewAccumulator)
+        in
+            Result,
+
+    AllData = GetAllPages(null, {}),
+    ToTable = Table.FromRecords(AllData)
+in
+    ToTable
+```
+
+### Link-Based Pagination (HATEOAS)
+
+```m
+let
+    // Function to follow next links
+    GetAllPages = (url as text, accumulator as list) =>
+        let
+            Response = Json.Document(Web.Contents(url)),
+            CurrentData = Response[results],
+            NextUrl = try Response[next] otherwise null,
+            NewAccumulator = accumulator & CurrentData,
+
+            Result = if NextUrl = null then
+                NewAccumulator
+            else
+                @GetAllPages(NextUrl, NewAccumulator)
+        in
+            Result,
+
+    StartUrl = "https://api.example.com/data",
+    AllData = GetAllPages(StartUrl, {}),
+    ToTable = Table.FromRecords(AllData)
+in
+    ToTable
+```
+
+---
+
+## POST Requests
+
+### POST with JSON Body
+
+```m
+let
+    Url = "https://api.example.com/query",
+
+    // Request body
+    Body = Json.FromValue([
+        startDate = "2024-01-01",
+        endDate = "2024-12-31",
+        filters = [
+            status = "active",
+            region = "US"
+        ]
+    ]),
+
+    Source = Json.Document(
+        Web.Contents(
+            Url,
+            [
+                Headers = [
+                    #"Content-Type" = "application/json"
+                ],
+                Content = Body
+            ]
+        )
+    )
+in
+    Source
+```
+
+### POST with Form Data
+
+```m
+let
+    Url = "https://api.example.com/token",
+
+    // URL-encoded form data
+    FormData = Text.ToBinary(
+        Uri.BuildQueryString([
+            grant_type = "client_credentials",
+            client_id = "your-client-id",
+            client_secret = "your-client-secret"
+        ])
+    ),
+
+    Source = Json.Document(
+        Web.Contents(
+            Url,
+            [
+                Headers = [
+                    #"Content-Type" = "application/x-www-form-urlencoded"
+                ],
+                Content = FormData
+            ]
+        )
+    )
+in
+    Source
+```
+
+---
+
+## Error Handling
+
+### Basic Error Handling
+
+```m
+let
+    SafeApiCall = (url as text) =>
+        let
+            Response = try Json.Document(Web.Contents(url))
+        in
+            if Response[HasError] then
+                [
+                    success = false,
+                    error = Response[Error][Message],
+                    data = null
+                ]
+            else
+                [
+                    success = true,
+                    error = null,
+                    data = Response[Value]
+                ],
+
+    Result = SafeApiCall("https://api.example.com/data")
+in
+    Result
+```
+
+### Retry with Backoff
+
+```m
+let
+    // Retry function with exponential backoff
+    RetryRequest = (url as text, maxRetries as number, retryCount as number) =>
+        let
+            Response = try Json.Document(Web.Contents(url))
+        in
+            if Response[HasError] then
+                if retryCount >= maxRetries then
+                    error Response[Error]
+                else
+                    let
+                        // Wait before retry (simplified - actual delay not supported in M)
+                        _ = Function.InvokeAfter(
+                            () => null,
+                            #duration(0, 0, 0, Number.Power(2, retryCount))
+                        )
+                    in
+                        @RetryRequest(url, maxRetries, retryCount + 1)
+            else
+                Response[Value],
+
+    Result = RetryRequest("https://api.example.com/data", 3, 0)
+in
+    Result
+```
+
+### Handle HTTP Status Codes
+
+```m
+let
+    Url = "https://api.example.com/data",
+
+    Response = Web.Contents(
+        Url,
+        [
+            ManualStatusHandling = {400, 401, 403, 404, 500}
+        ]
+    ),
+
+    Metadata = Value.Metadata(Response),
+    StatusCode = Metadata[Response.Status],
+
+    Result = if StatusCode = 200 then
+        Json.Document(Response)
+    else if StatusCode = 401 then
+        error Error.Record("Authentication Error", "Invalid credentials")
+    else if StatusCode = 404 then
+        error Error.Record("Not Found", "Resource not found")
+    else if StatusCode >= 500 then
+        error Error.Record("Server Error", "API server error: " & Text.From(StatusCode))
+    else
+        error Error.Record("API Error", "Unexpected status: " & Text.From(StatusCode))
+in
+    Result
+```
+
+---
+
+## Rate Limiting
+
+### Delay Between Requests
+
+```m
+let
+    BaseUrl = "https://api.example.com/data",
+    Endpoints = {"users", "orders", "products"},
+
+    // Function with delay
+    GetWithDelay = (endpoint as text, index as number) =>
+        let
+            // Delay increases with index
+            _ = Function.InvokeAfter(
+                () => null,
+                #duration(0, 0, 0, index * 0.5)  // 0.5 second delay per request
+            ),
+            Response = Json.Document(
+                Web.Contents(BaseUrl & "/" & endpoint)
+            )
+        in
+            Response,
+
+    // Process with index for delay
+    Results = List.Transform(
+        List.Zip({Endpoints, {0..List.Count(Endpoints)-1}}),
+        each GetWithDelay(_{0}, _{1})
+    )
+in
+    Results
+```
+
+### Respect Retry-After Header
+
+```m
+let
+    Url = "https://api.example.com/data",
+
+    Response = Web.Contents(
+        Url,
+        [ManualStatusHandling = {429}]
+    ),
+
+    Metadata = Value.Metadata(Response),
+    StatusCode = Metadata[Response.Status],
+
+    Result = if StatusCode = 429 then
+        let
+            RetryAfter = try Number.From(Metadata[Headers][#"Retry-After"]) otherwise 60,
+            _ = Function.InvokeAfter(() => null, #duration(0, 0, 0, RetryAfter))
+        in
+            Json.Document(Web.Contents(Url))  // Retry
+    else
+        Json.Document(Response)
+in
+    Result
+```
+
+---
+
+## Common API Patterns
+
+### REST API with Nested JSON
+
+```m
+let
+    Source = Json.Document(
+        Web.Contents("https://api.example.com/orders")
+    ),
+
+    // Expand nested records
+    Orders = Source[data],
+    ToTable = Table.FromRecords(Orders),
+
+    // Expand nested customer object
+    ExpandedCustomer = Table.ExpandRecordColumn(
+        ToTable, "customer",
+        {"id", "name", "email"},
+        {"customer_id", "customer_name", "customer_email"}
+    ),
+
+    // Expand nested line items list
+    ExpandedItems = Table.ExpandListColumn(ExpandedCustomer, "line_items"),
+    ExpandedItemDetails = Table.ExpandRecordColumn(
+        ExpandedItems, "line_items",
+        {"product_id", "quantity", "price"},
+        {"item_product_id", "item_quantity", "item_price"}
+    )
+in
+    ExpandedItemDetails
+```
+
+### GraphQL API
+
+```m
+let
+    Url = "https://api.example.com/graphql",
+
+    Query = "
+        query GetOrders($startDate: String!) {
+            orders(startDate: $startDate) {
+                id
+                date
+                total
+                customer {
+                    name
+                    email
+                }
+            }
+        }
+    ",
+
+    Variables = [startDate = "2024-01-01"],
+
+    Body = Json.FromValue([
+        query = Query,
+        variables = Variables
+    ]),
+
+    Response = Json.Document(
+        Web.Contents(
+            Url,
+            [
+                Headers = [#"Content-Type" = "application/json"],
+                Content = Body
+            ]
+        )
+    ),
+
+    Data = Response[data][orders],
+    ToTable = Table.FromRecords(Data)
+in
+    ToTable
+```
+
+### OData API
+
+```m
+let
+    // OData is well-supported natively
+    Source = OData.Feed(
+        "https://services.odata.org/V4/Northwind/Northwind.svc/",
+        null,
+        [
+            Query = [
+                #"$filter" = "Country eq 'USA'",
+                #"$select" = "CustomerID,CompanyName,City",
+                #"$top" = "100"
+            ]
+        ]
+    )
+in
+    Source
+```
+
+---
+
+## Best Practices
+
+### 1. Use Parameters for URLs and Keys
+
+```m
+let
+    // Reference parameters instead of hardcoding
+    BaseUrl = BaseUrlParameter,
+    ApiKey = ApiKeyParameter,
+
+    Source = Json.Document(
+        Web.Contents(
+            BaseUrl,
+            [Headers = [#"Authorization" = "Bearer " & ApiKey]]
+        )
+    )
+in
+    Source
+```
+
+### 2. Enable Query Folding Where Possible
+
+```m
+let
+    // Use RelativePath for consistent base URL
+    // This helps with caching and query folding
+    Source = Json.Document(
+        Web.Contents(
+            "https://api.example.com",
+            [RelativePath = "/data/orders"]
+        )
+    )
+in
+    Source
+```
+
+### 3. Handle Empty Responses
+
+```m
+let
+    Response = Json.Document(Web.Contents(Url)),
+    Data = try Response[data] otherwise {},
+    ToTable = if Data = {} or Data = null then
+        #table({"Column1"}, {})  // Empty table with schema
+    else
+        Table.FromRecords(Data)
+in
+    ToTable
+```
+
+### 4. Log API Calls (Debug)
+
+```m
+let
+    Url = "https://api.example.com/data",
+
+    // Log the URL being called
+    _ = Diagnostics.Trace(
+        TraceLevel.Information,
+        "Calling API: " & Url,
+        () => true,
+        true
+    ),
+
+    Response = Json.Document(Web.Contents(Url))
+in
+    Response
+```
+
+---
+
+## Troubleshooting
+
+| Issue | Cause | Solution |
+|-------|-------|----------|
+| "Access denied" | CORS or auth issue | Check credentials, use Web.Contents options |
+| "Formula firewall" | Privacy levels | Set privacy to Public/Organizational |
+| Timeout | Slow API | Increase timeout, paginate smaller |
+| "Invalid JSON" | Not JSON response | Check response content-type |
+| Rate limited | Too many requests | Add delays, implement retry |
+
+---
+
+## Security Checklist
+
+- [ ] Never hardcode credentials in queries
+- [ ] Use parameters for API keys
+- [ ] Store secrets in Azure Key Vault or parameter
+- [ ] Review data gateway requirements
+- [ ] Test with minimum required permissions
+- [ ] Document API rate limits