fiftyone.geolocation 4.3.1 → 4.3.6

package/ci/common-ci/scripts/modules/AzurePortalModules/51DChecksModule/51DChecksModule.psm1

@@ -0,0 +1,68 @@
+<#
+  ==================== Checks =====================
+  .Description
+  This module contains functions to check various aspects of Azure Resources.
+#>
+
+
+# Check if a service is available by polling an Uri. Returns the status code 
+# recevied after a number of retrys. An interval can be specified to wait 
+# between retries.
+function Get-UriHTTPStatus {
+    param (
+        # The URI of the service to test, e.g. https://51degrees.com
+        [Parameter(Mandatory=$true)]
+        [string]$Uri,
+
+        # The number of tries before aborting the test.
+        [Parameter(Mandatory=$true)]
+        [int]$NumberOfTries,
+
+        # The delay between tries in seconds.
+        [Parameter(Mandatory=$true)]
+        [int]$SecondsBetweenTries
+    )
+
+    Write-Host -NoNewline "Waiting for response from $Uri"
+
+    $Tries = 0
+    $HTTP_Status = $null
+    While ($HTTP_Status -ne 200 -And $Tries -le $NumberOfTries) {
+
+        # Wait before querying Uri
+        Start-Sleep -Seconds $SecondsBetweenTries
+
+        # Get HTTP status code from invoking a request for the Uri.
+        try{
+            $HTTP_Status = (Invoke-WebRequest -Uri $Uri -UseBasicParsing -DisableKeepAlive).StatusCode
+        }
+        catch [Net.WebException]
+        {
+            # If there is a Web Exception then get the status code
+            $HTTP_Status = [int]$_.Exception.Response.StatusCode
+        }
+        catch [Exception]
+        {
+            # If there is a general exception then assume a status of null
+            # (no response).
+            $HTTP_Status = $null
+        }
+
+        $Tries = $Tries +1
+        Write-Host -NoNewLine "."
+    }
+    
+    Write-Host ""
+
+    # Return status code if service is not accepting requests else return 0
+    if ($HTTP_Status -ne 200 -or $null -eq $HTTP_Status) {
+        Write-Host "Service is not accepting requests for Uri '$($Uri)'"
+        return $HTTP_Status
+    } else {
+        Write-Host "Service is responding for Uri '$($Uri)'"
+        return 0
+    }
+   
+}
+
+Export-ModuleMember -Function Get-UriHTTPStatus

package/ci/common-ci/scripts/modules/AzurePortalModules/51DCloudModule/51DCloudModule.psm1

@@ -0,0 +1,155 @@
+<#
+  ==================== Cloud =====================
+  .Description
+  This module contains functions specific to the cloud environment configuration
+#>
+
+# Create a new blocklist json file and add it to the provided storage account
+function Add-BlocklistFile {
+
+    param (
+        [Parameter(Mandatory=$true)]
+        [Object]$storageAccount, # storage account object
+
+        [string]$shareName = "cloudconfig", # file share name
+        [string]$fileName = "blocklist.json" # blocklist filename
+    )
+
+    # Storage context
+    $ctx = $storageAccount.Context
+
+    $storageFiles = Get-AzStorageFile `
+        -ShareName $shareName `
+        -Context $ctx
+
+    $storageFile = $storageFiles | Where-Object {$_.Name -eq $fileName }
+
+    # If the blocklist file does not exist then create a new one.
+    if(-not $storageFile) {
+        Write-Host "Adding blocklist file..."
+
+        # Create the blocklist json object
+        $blocklist = @{
+            DataPublishedDateTime = (Get-Date).ToString("yyyy-MM-dd")
+            UpdateAvailableTime = (Get-Date).ToString("yyyy-MM-dd")
+            BlacklistedKeys = @()
+            BlacklistedReferers = @()
+            BlacklistedIPs = @()
+            BlacklistedLicenseIds = @()
+            BlacklistedResourceKeys = @()
+        }
+
+        # Convert the object to JSON.
+        $content = $blocklist | ConvertTo-Json -depth 32
+
+        # Create a file with the JSON content on disk.
+        $Null = New-Item . -Name $fileName -ItemType "file" -Value $content -Force
+
+        # Try to get the file share
+        $share = $null
+        try {
+            $share = Get-AzStorageShare `
+                -Name $shareName `
+                -Context $ctx `
+                -ErrorAction SilentlyContinue
+        } catch {}
+
+        # Create the file share if it does not exist.
+        if(-not($share)) {
+            $share = New-AzStorageShare `
+                -Name $shareName `
+                -Context $ctx
+        }
+    
+        # Add the file to the file share.
+        $Null = Set-AzStorageFileContent `
+            -ShareName $shareName `
+            -Source $fileName `
+            -Context $ctx
+
+        # Cleanup the file on disk.
+        $Null = Remove-Item $fileName
+    }
+
+    # Create the shared access signature URI for accessing the file.
+    $StartTime = Get-Date
+    $EndTime = $StartTime.AddYears(50)
+
+    $token = New-AzStorageFileSASToken `
+        -ShareName $shareName `
+        -Path $fileName `
+        -Permission "r" `
+        -StartTime $StartTime `
+        -ExpiryTime $EndTime `
+        -Context $ctx
+
+    $sas = "$($ctx.FileEndPoint)$($shareName)/$($fileName)$($token)"
+
+    return $sas
+}
+
+# Add a Resource key to the entitlement table of a storage account.
+function Add-ResourceKey {
+    param (
+        [Parameter(Mandatory=$true)]
+        [Object]$storageAccount
+    )
+
+    Write-Host "Setting up test resource key..."
+
+    # Make sure the AzTable module is installed
+    if (-not (Get-Module -ListAvailable -Name AzTable)) {
+        Write-Host "Az Table module does not exist, installing..."
+        Install-Module AzTable
+    }
+    
+    # Construct the resource key table row
+    $resource = @{
+        partitionKey = "AQS5HKcyxmoxU0-q10g"
+        rowKey = "2020-02-05T15:23:17.1100358Z"
+        properties = @{
+            "AgreedToTerms"=$TRUE
+            "Domains"=""
+            "Email"="engineering@51degrees.com"
+            "ProductKeys"="FBVAAAELPALWBBNCEJAGALSAB2ASUP34ZHMR3475Y9SL92L2P7A5X6HUB8JYQFBC7GDNY9AYFYFS5VTYXTB8BNRCA5KXE"
+            "Properties"=""
+            "ReceiveMarketingEmails"=$FALSE
+        }
+    }
+
+    # Storage Account context
+    $ctx = $storageAccount.Context
+
+    # The name of the storage table
+    $name = "entitlement"
+
+    # Try to get the storage table.
+    $t = $null
+    try {
+        $t = Get-AzStorageTable `
+            -Name $name `
+            -Context $ctx `
+            -ErrorAction SilentlyContinue
+    } catch { }
+
+    # If the storage table sodes not exist then create it.
+    if (-not ($t)) {
+        $t = New-AzStorageTable -Name $name -Context $ctx
+    }
+        
+    # Try to get the exisiting Resource Key.
+    $table = $t.CloudTable
+    $row = Get-AzTableRow -table $table -partitionKey $resource.partitionKey | Format-Table
+
+    # If it does not exist then create the Resource Key Row in the table.
+    if(-not ($row)) {
+        Add-AzTableRow `
+            -table $table `
+            -partitionKey $resource.partitionKey `
+            -rowKey $resource.rowKey `
+            -property $resource.properties
+    }
+
+    # Log the key to console.
+    Write-Host "Resource key for testing: $($resource.partitionKey)"
+}

package/ci/common-ci/scripts/modules/AzurePortalModules/51DConnectAzureModule/51DConnectAzureModule.psm1

@@ -0,0 +1,45 @@
+<#
+  ==================== ConnectAzure =====================
+  .Description
+  This module contains functions to connect to an Azure Account and select an
+  Azure subscription.
+#>
+
+function Connect-AzSubscription {
+
+    $subscriptionId=''
+    $ctx = $null
+
+    # Connect to the Azure remote management API. Authentication may be required.
+    Read-Host -Prompt ("You will be prompted to log in to your Microsoft work account. " +
+        "Please choose an account which has access to create new Azure resources. " +
+        "Press enter to continue...")
+    Connect-AzAccount
+
+    $subs = Get-AzSubscription 
+
+    if ($subs.Length -eq 0) {
+        Write-Host ("No Azure subscriptions for this account could be found." + 
+            "Please confirm this account has access to azure and try again.")
+        exit 1
+    }
+
+    $subs | Format-Table
+    
+    while ($null -eq $ctx) {
+        while ('' -eq $subscriptionId) {
+            $subscriptionId = Read-Host -Prompt ("Choose the azure subscription id from "+ 
+            "above and paste on this line, then press enter to continue")
+        }
+
+        # Set the context to the subscription Id where the cluster will be created
+        try {
+            $ctx = Select-AzSubscription -SubscriptionId $subscriptionId -ErrorAction Stop
+        } 
+        catch {
+            Write-Host "Invalid subscription selected."
+        }
+    }
+}
+
+Export-ModuleMember -Function Connect-AzSubscription 

package/ci/common-ci/scripts/modules/AzurePortalModules/51DSSLBindingsModule/51DSSLBIndingsModule.psm1

@@ -0,0 +1,100 @@
+<#
+  ==================== SSLBindings =====================
+  .Description
+  This module contains a function to refresh the custom domains assigned to an
+  Azure app service and update the SSL bindings.
+
+  References:
+    https://github.com/uglide/azure-content/blob/master/articles/app-service-web/app-service-web-app-powershell-ssl-binding.md
+    https://docs.microsoft.com/en-us/powershell/module/Az.websites/remove-Azwebappsslbinding?view=Azps-6.13.0
+#>
+
+# Update hosts and SSL bindings for a web app.
+function Set-SSLBindings {
+
+    param(
+        [Parameter(Mandatory=$true)]
+        [Hashtable[]]$Certs,
+
+        [Parameter(Mandatory=$true)]
+        [Hashtable[]]$Hosts,
+
+        [Parameter(Mandatory=$true)]
+        [Object]$Webapp
+    )
+
+    # Remove all the exising SSL bindings.
+    foreach ($hostName in $Hosts.Name) {
+        Remove-AzWebAppSSLBinding `
+            -ResourceGroupName $Webapp.ResourceGroup `
+            -WebAppName $Webapp.Name `
+            -Name $hostName `
+            -Force `
+            -DeleteCertificate $false 
+    }
+
+    # Replace all the existing host names with the ones in this script.
+    Set-AzWebApp `
+        -ResourceGroupName $Webapp.ResourceGroup `
+        -Name $Webapp.Name `
+        -HostNames @($Hosts.Name, $Webapp.DefaultHostName) `
+        -Use32BitWorkerProcess $false 
+
+    # # Upload the new SSL certificates if they do not exist and bind them to the 
+    # # host name. If the SSL certificate does exist then use the existing one.
+    foreach($current in $Hosts) {
+
+        # Use the -like directive to find wildcard certificates that have the
+        # *.domain.com format.
+        Write-Host "Finding certificates for $($current.Name)"
+        $existing = New-Object -TypeName 'System.Collections.ArrayList'
+        foreach($cert in (Get-AzWebAppCertificate `
+            -ResourceGroupName $Webapp.ResourceGroup) | `
+            Where-Object { `
+                ($_.Location -eq $Webapp.Location) -and `
+                ($_.ExpirationDate -gt (Get-Date))}) {
+            foreach($hostName in $cert.HostNames) {
+                if ($current.Name -like $hostName) {
+                    $existing.Add($cert) | Out-Null
+                    Write-Host "Found" $cert.Thumbprint
+                    Break
+                }
+            }
+        }
+        
+        if (($null -eq $existing) -OR ($existing.Count -eq 0)) {
+
+            # There are no available certificates. Find the one that is contained
+            # in the same folder as this script and add it to the web app.
+            $cert = $Certs |
+                Where-Object CertificateFile -eq $current.CertificateFile |
+                Select-Object -First 1
+            if ($null -ne $cert) {
+                Write-Host "Adding certificate for $($current.Name)"
+                New-AzWebAppSSLBinding `
+                    -ResourceGroupName $Webapp.ResourceGroup `
+                    -WebAppName $Webapp.Name `
+                    -Name $current.Name `
+                    -CertificateFilePath (Join-Path $pwd $cert.CertificateFile) `
+                    -CertificatePassword $cert.Password 
+            } else {
+                Write-Host "No cerificate available for $($current.Name)"
+            }
+        } else {
+
+            # Order the list so that the certificate that expires the furthest in the
+            # future is used form the domain.
+            $existing = $existing | Sort-Object ExpirationDate -Descending
+
+            # Use the thumbprint from the certicate to set bind to the domain.
+            Write-Host "Using existing certificate $($existing[0].Thumbprint) for $($current.Name)"
+            New-AzWebAppSSLBinding `
+                -ResourceGroupName $Webapp.ResourceGroup `
+                -WebAppName $Webapp.Name `
+                -Name $current.Name `
+                -Thumbprint $existing[0].Thumbprint
+        }
+    }
+}
+
+Export-ModuleMember -Function Set-SSLBindings

package/ci/common-ci/scripts/modules/AzurePortalModules/51DStagingSlotModule/51DStagingSlotModule.psm1

@@ -0,0 +1,50 @@
+<#
+  ==================== Staging Slots =====================
+  .Description
+  This module contains a function to refresh or add the staging slot on an 
+  Azure App Service.
+#>
+
+
+function Add-WebAppStagingSlot {
+
+    param(
+        [Parameter(Mandatory=$true)]
+        [Object]$Webapp,
+
+        [Hashtable]$AppSettings
+    )
+
+    Write-Host "Adding Staging slot..."
+
+    # Remove the staging slot ready to rebuild it.
+    $null = Remove-AzWebAppSlot `
+      -ResourceGroupName $webapp.ResourceGroup `
+      -Name $webapp.Name `
+      -Slot "Staging" `
+      -Force
+
+    # Adds a new staging slot with fresh configuration.
+    $null = New-AzWebAppSlot `
+      -ResourceGroupName $webapp.ResourceGroup `
+      -Name $webapp.Name `
+      -Slot "Staging"
+
+    if ($null -eq $AppSettings){
+      $AppSettings = $webapp.SiteConfig.AppSettings
+    }
+
+    # Config the fresh staging slot. Belt and braces incase the CD script doesn't
+    # set the correct parameters.
+    $null = Set-AzWebAppSlot `
+      -ResourceGroupName $webapp.ResourceGroup `
+      -Name $webapp.Name `
+      -Slot "Staging" `
+      -AppSettings $AppSettings `
+      -WebSocketsEnabled $false `
+      -Use32BitWorkerProcess $false `
+      -AlwaysOn $true `
+      -FtpsState "Disabled"
+}
+
+Export-ModuleMember -Function Add-WebAppStagingSlot

package/ci/common-ci/scripts/modules/AzurePortalModules/51DStorageAccountModule/51DStorageAccountModule.psm1

@@ -0,0 +1,248 @@
+<#
+  ==================== Stroage Accounts =====================
+  .Description
+  This module contains a function to create storage accounts.
+  References:
+    https://docs.microsoft.com/en-us/azure/cosmos-db/scripts/powershell/table/create
+    https://docs.microsoft.com/en-us/powershell/module/az.cosmosdb/new-azcosmosdbaccount
+#>
+
+# Create a new Storage Account
+function New-StorageAcc {
+  param (
+    [Parameter(Mandatory = $true)]
+    [string]$storageAccName,
+
+    [Parameter(Mandatory = $true)]
+    [string]$resourceGroupName,
+
+    [Parameter(Mandatory = $true)]
+    [string]$region,
+
+    [Hashtable]$tags = @{}
+  )
+  Write-Host ""
+
+  while ($storageAccName.length -lt 3) {
+    $choice = Read-Host -Prompt ("Storage account name '$($storageAccName)' " + 
+      "is too short, enter a new name here")
+    if (-not [string]::IsNullOrWhiteSpace($choice)) {
+      $storageAccName = $choice.Replace("-", "").ToLower()
+    }
+  }
+
+  while ($storageAccName.length -gt 24) {
+    $truncated = $storageAccName.Substring(0, 24)
+    $choice = Read-Host -Prompt ("Storage account name '$($storageAccName)' " + 
+      "is too long, use trunctated name or enter a new name here [$($truncated)]")
+    if ([string]::IsNullOrWhiteSpace($choice)) {
+      $storageAccName = truncated
+    }
+    else {
+      $storageAccName = $choice.Replace("-", "").ToLower()
+    }
+  }
+
+  $sb = $null
+  
+  while (-not $sb) {
+    # Check if storage account exists
+    $sb = Get-AzStorageAccount `
+      -ResourceGroupName $resourceGroupName `
+      -Name $storageAccName `
+      -ErrorAction SilentlyContinue
+
+    if(-not $sb) {
+      # create sotrage account
+      Write-Host "Creating storage account:" $storageAccName
+
+      $sb = New-AzStorageAccount `
+        -ResourceGroupName $resourceGroupName `
+        -Name $storageAccName `
+        -Location $region `
+        -Tag $tags `
+        -SkuName Standard_RAGRS `
+        -Kind StorageV2
+
+      if(-not $sb) {
+        # If there is still no storage account then the 
+        # 'New-AzStorageAccount' command has failed. Retreive the last
+        # Azure Error.
+        $lastError = Resolve-AzError -Last -WarningAction silentlyContinue
+
+        # Check that the last error is from the expected source, if not
+        # we can't handle it. Prompt the user to check the Errors 
+        # manually.
+        if ($lastError.Exception.Source -ne "Microsoft.Azure.PowerShell.Cmdlets.Storage.Management") {
+          Write-Host ("Could not resolve error creating storage " +
+            "account. Use the PowerShell Cmdlet 'Resolve-AzError -Last' to " + 
+            "inspect the last Azure error in this session.")
+          exit 1
+        }
+
+        # If the stroage account name has already been taken then
+        # prompt the user for a new account name.
+        if ($lastError.Message.Contains("The storage account named " +
+          "$($storageAccName) is already taken. (Parameter 'Name')")) {
+          $storageAccName = Read-Host -Prompt ("Storage account name " +
+            "'$($storageAccName)' already taken in a different Azure " + 
+            "subscription, enter a new name here")
+        } elseif ($lastError.Message.Contains("$($storageAccName) is not a " +
+          "valid storage account name.")) {
+          $storageAccName = Read-Host -Prompt ("'$($storageAccName)' is not a " +
+            "valid storage account name, enter a new name here")
+          # If the Azure error is not the expected error then abort
+          # the process and ask the user to review the Azure errors 
+          # manually.
+        } else {
+          Write-Host "Error message: $($lastError.Message)"
+          Write-Host ("Could not resolve error creating storage " +
+            "account. Use the PowerShell Cmdlet 'Resolve-AzError -Last' to " + 
+            "inspect the last Azure error in this session.")
+          exit 1
+        }
+      }
+    } else {
+      Write-Host -ForegroundColor Magenta ("Storage account '$($storageAccName)' " +
+        "already exists, using exsiting account")  
+    }
+  }
+
+  return $sb
+}
+
+# Function to create a new Cosmos DB account. First check if one with the same
+# account name exists already, if not the create a new one, otherwise prompt the
+# user for a new name.
+function New-CosmosAccount {
+    
+  param ( 
+    # The name of the Cosmos DB Account
+    [Parameter(Mandatory = $true)]
+    [string]$accountName,
+
+    # The name of the resource group where the Cosmos DB account is created
+    # or located
+    [Parameter(Mandatory = $true)]
+    [string]$resourceGroupName,
+
+    # Tags to create the Cosmos DB account with.
+    [hashtable]$tags = @{},
+
+    # The kind of CosmosDB API to use.
+    [string]$apiKind = "Table",
+    [string]$consistencyLevel = "BoundedStaleness",
+    [int]$maxStalenessIntervalInSeconds = 86400,
+    [int]$maxStalenessPrefix = 1000000,
+    [Array]$locations = @(
+      (New-AzCosmosDBLocationObject -LocationName "West Europe" -FailoverPriority 0 -IsZoneRedundant 0),
+      (New-AzCosmosDBLocationObject -LocationName "Central US" -FailoverPriority 1 -IsZoneRedundant 0)
+      # TODO consider adding central india as a cosmos db region.
+      #(New-AzCosmosDBLocationObject -LocationName "Central India" -FailoverPriority 1 -IsZoneRedundant 0)
+    )
+  )
+
+  $cosmosDBAccount = $null 
+
+  while (-not $cosmosDBAccount) {
+    # Get an existing Cosmos DB account.
+    $cosmosDBAccount = Get-AzCosmosDBAccount `
+      -ResourceGroupName $resourceGroupName `
+      -Name $accountName `
+      -ErrorAction SilentlyContinue
+
+    if (-not $cosmosDBAccount) {
+      # If there is no Cosmos DB account then try to create one.
+      Write-Host "Creating Cosmos DB account '$($accountName)'"
+      $cosmosDBAccount = New-AzCosmosDBAccount `
+        -ResourceGroupName $resourceGroupName `
+        -Name $accountName `
+        -LocationObject $locations `
+        -ApiKind $apiKind `
+        -DefaultConsistencyLevel $consistencyLevel `
+        -MaxStalenessIntervalInSeconds $maxStalenessIntervalInSeconds `
+        -MaxStalenessPrefix $maxStalenessPrefix `
+        -Tag $tags
+
+      if (-not $cosmosDBAccount) {
+        # If there is still no Cosmos DB account then the 
+        # 'New-AzCosmosDBAccount' command has failed. Retreive the last
+        # Az Error.
+        $lastError = Resolve-AzError -Last -WarningAction silentlyContinue
+
+        # Check that the last error is from the expected source, if not
+        # we can't handle it. Prompt the user to check the Errors 
+        # manually.
+        if ($lastError.Exception.Source -ne "Microsoft.Azure.Management.CosmosDB") {
+          Write-Host ("Could not resolve error creating cosmos DB " +
+            "account. Use the PowerShell Cmdlet 'Resolve-AzError -Last' to " + 
+            "inspect the last Azure error in this session.")
+          exit 1
+        }
+
+        # The Cosmos DB DNS record is determined by the account name. 
+        # If the error is caused by the DNS record being taken then
+        # prompt the user for a new account name.
+        if ($lastError.Message.Contains("Dns record for " +
+            "$($accountName) under zone Document is already " +
+            "taken. Please use a different name for the account")) {
+          $suggestion = $accountName + "-1"
+          $choice = Read-Host -Prompt ("Cosmos DB account name " +
+            "'$($accountName)' already taken in a different Azure " +
+            "Subscription, use suggestion or enter a new name here [$($suggestion)]")
+
+          if ([string]::IsNullOrWhiteSpace($choice)) {
+            $accountName = $suggestion
+          }
+          else {
+            $accountName = $choice
+          }
+          # If the Cosmos DB error is not the expected error then abort
+          # the process and ask the user to review the Azure errors 
+          # manually.
+        }
+        else {
+          Write-Host "Error message: $($lastError.Message)"
+          Write-Host ("Could not resolve error creating cosmos DB " +
+            "account. Use the PowerShell Cmdlet 'Resolve-AzError -Last' to " + 
+            "inspect the last Azure error in this session.")
+          exit 1
+        }
+      }   
+    } else {
+      Write-Host -ForegroundColor Magenta ("Cosmos DB account '$($accountName)' " +
+        "exists, using existing account.")
+    }
+  }
+
+  return $cosmosDBAccount
+}
+
+function New-CosmosDbTable {
+
+  param (
+    [Parameter(Mandatory=$true)]
+    [object]$cosmosAccount,
+
+    [Parameter(Mandatory=$true)]
+    [string]$tableName,
+
+    [Parameter(Mandatory=$true)]
+    [int]$tableRUs
+  )
+
+  $table = Get-AzCosmosDBTable `
+    -Name $tableName `
+    -ParentObject $cosmosAccount `
+    -ErrorAction SilentlyContinue
+
+  if(-not $table) {
+    Write-Host "Creating Cosmos DB table: $($tableName)"
+    New-AzCosmosDBTable `
+      -ParentObject $cosmosAccount `
+      -Name $tableName `
+      -Throughput $tableRUs
+  } else {
+    Write-Host "Cosmos DB table '$($tableName)' already exists."
+  }
+}