@orcabus/platform-cdk-constructs 0.0.5 → 0.0.7-alpha.3

package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/metadata/subject_helpers.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+
+
+"""
+Helper functions for a subject
+"""
+
+# Standard imports
+from typing import List
+from requests import HTTPError
+
+# Local imports
+from . import get_metadata_request_response_results, get_item_objs_from_item_id_list
+from .errors import SubjectNotFoundError
+from .globals import SUBJECT_ENDPOINT, ORCABUS_ULID_REGEX_MATCH
+from .models import Subject, Sample, LibraryDetail
+
+
+def get_subject_from_subject_id(subject_id: str) -> Subject:
+    """
+    Get subject from the subject id
+    :param subject_id:
+    :return:
+    """
+    # We have an internal id, convert to int
+    params = {
+        "subject_id": subject_id
+    }
+
+    # Get subject
+    try:
+        query_results = get_metadata_request_response_results(SUBJECT_ENDPOINT, params)
+        assert len(query_results) == 1
+        return query_results[0]
+    except (HTTPError, AssertionError):
+        raise SubjectNotFoundError(
+            subject_id=subject_id
+        )
+
+
+def get_subjects_list_from_subject_id_list(subject_id_list: List[str], accept_missing: bool = False) -> List[Sample]:
+    """
+    Get sample from the sample id
+    :param sample_id:
+    :return:
+    """
+    # We have an internal id, convert to int
+    return list(map(
+        lambda sample_iter_: Sample(**sample_iter_),
+        get_item_objs_from_item_id_list(
+            item_id_list=subject_id_list,
+            item_identifier="subjectId",
+            endpoint=SUBJECT_ENDPOINT,
+            accept_missing=accept_missing
+        )
+    ))
+
+
+def coerce_subject_id_or_orcabus_id_to_subject_orcabus_id(id_: str) -> str:
+    if ORCABUS_ULID_REGEX_MATCH.match(id_):
+        return id_
+    else :
+        return get_subject_orcabus_id_from_subject_id(id_)
+
+
+def get_subject_orcabus_id_from_subject_id(subject_id: str) -> str:
+    """
+    Get the subject orcabus id from the subject id
+    :param subject_id:
+    :return:
+    """
+    return get_subject_from_subject_id(subject_id)['orcabusId']
+
+
+def get_subjects_list_from_subject_orcabus_id_list(subject_orcabus_id_list: List[str], accept_missing: bool = False) -> List[Sample]:
+    """
+    Get sample from the sample id
+    :param sample_id:
+    :return:
+    """
+    # We have an internal id, convert to int
+    return list(map(
+        lambda sample_iter_: Sample(**sample_iter_),
+        get_item_objs_from_item_id_list(
+            item_id_list=subject_orcabus_id_list,
+            item_identifier="subjectId",
+            endpoint=SUBJECT_ENDPOINT,
+            accept_missing=accept_missing
+        )
+    ))
+
+
+def get_subject_from_subject_orcabus_id(subject_orcabus_id: str) -> Subject:
+    """
+    Get subject from the subject id
+    :param subject_orcabus_id:
+    :return:
+    """
+    # Get subject id
+    # We have an internal id, convert to int
+    params = {
+        "orcabus_id": subject_orcabus_id
+    }
+
+    # Get subject
+    try:
+        query_results = get_metadata_request_response_results(SUBJECT_ENDPOINT, params)
+        assert len(query_results) == 1
+        return query_results[0]
+    except (HTTPError, AssertionError):
+        raise SubjectNotFoundError(
+            subject_orcabus_id=subject_orcabus_id
+        )
+
+
+def list_samples_in_subject(subject_orcabus_id: str) -> List[Sample]:
+    """
+    Given a subject id, list the samples in the subject
+    :param subject_orcabus_id:
+    :return:
+    """
+    from .sample_helpers import get_sample_from_sample_orcabus_id
+
+    # Get the subject
+    return list(map(
+        # For each subject, get libraries in subject
+        lambda library_iter_: get_sample_from_sample_orcabus_id(library_iter_['sample']['orcabusId']),
+        # Get list of subject orcabus ids
+        list_libraries_in_subject(subject_orcabus_id)
+    ))
+
+
+def list_libraries_in_subject(subject_orcabus_id: str) -> List[LibraryDetail]:
+    """
+    Given a subject id, return the list of library objects in the subject
+    :param subject_orcabus_id:
+    :return:
+    """
+    # Get ID For Subject
+    subject = get_subject_from_subject_orcabus_id(subject_orcabus_id)
+
+    # Get the subject
+    return subject.get("librarySet", [])
+
+
+def get_all_subjects() -> List[Subject]:
+    """
+    Get all subjects
+    :return:
+    """
+    return get_metadata_request_response_results(SUBJECT_ENDPOINT)

package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/sequence/__init__.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+
+from sequence_helpers import (
+    get_sequence_object_from_instrument_run_id,
+    get_sample_sheet_from_orcabus_id,
+    get_library_ids_in_sequence,
+)
+
+__all__ = [
+    "get_sequence_object_from_instrument_run_id",
+    "get_sample_sheet_from_orcabus_id",
+    "get_library_ids_in_sequence",
+]
+
+

package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/sequence/globals.py

@@ -0,0 +1,2 @@
+# Run name for sequence run manager
+SEQUENCE_SUBDOMAIN_NAME = 'sequence'

package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/sequence/models.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+
+from typing import TypedDict, Literal, NotRequired, Optional, Dict
+
+statusLiteral = Literal['STARTED', 'FAILED', 'SUCCEEDED', 'ABORTED', 'RESOLVED']
+
+
+class SequenceDetail(TypedDict):
+    orcabusId: str
+    instrumentRunId: str
+    experimentName: str
+    startTime: str
+    endTime: str
+    status: statusLiteral
+
+
+class Sequence(TypedDict):
+    orcabusId: str
+    libraries: str
+    sequenceRunId: str
+    status: statusLiteral
+    startTime: str
+    sampleSheetName: str
+    v1pre3Id: str
+    icaProjectId: str
+    apiUrl: str
+    endTime: str
+    runVolumeName: str
+    runFolderPath: str
+    runDataUri: str
+    instrumentRunId: str
+    reagentBarcode: str
+    flowcellBarcode: str
+    sequenceRunName: str
+    experimentName: str
+
+
+class SampleSheet(TypedDict):
+    orcabusId: str
+    sampleSheetName: str
+    associationStatus: str
+    associationTimestamp: str
+    sampleSheetContent: Optional[Dict]
+    sequence: str

package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/sequence/sequence_helpers.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+from .globals import SEQUENCE_SUBDOMAIN_NAME
+from .models import Sequence, SequenceDetail, SampleSheet
+from ..utils.requests_helpers import get_request_response_results, get_request, get_url
+
+
+def get_sequence_url(endpoint: str) -> str:
+    """
+    Get the URL for the Metadata endpoint
+    :param endpoint:
+    :return:
+    """
+    return get_url(
+        endpoint,
+        SEQUENCE_SUBDOMAIN_NAME
+    )
+
+def get_sequence_object_from_instrument_run_id(instrument_run_id: str) -> SequenceDetail:
+    """
+    Get the sequence object from the instrument run id.
+    :param instrument_run_id:
+    :return:
+    """
+
+    return Sequence(
+        **dict(
+            get_request_response_results(
+                get_sequence_url(endpoint="api/v1/sequence"),
+                params={
+                    "instrumentRunId": instrument_run_id,
+                }
+            )[0]
+        )
+    )
+
+
+def get_sample_sheet_from_orcabus_id(sequence_orcabus_id: str) -> SampleSheet:
+    """
+    Get the sample sheet from the orcabus id.
+    :param sequence_orcabus_id:
+    :return:
+    """
+
+    return SampleSheet(
+        **dict(
+            get_request(
+                get_sequence_url(endpoint=f"api/v1/sequence/{sequence_orcabus_id}/sample_sheet")
+            )
+        )
+    )
+
+
+def get_library_ids_in_sequence(sequence_orcabus_id: str) -> list[str]:
+    """
+    Get the library ids in the sequence run.
+    :param sequence_orcabus_id:
+    :return:
+    """
+
+    return get_request(
+        get_sequence_url(endpoint=f"api/v1/sequence/{sequence_orcabus_id}")
+    )['libraries']

package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/utils/aws_helpers.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+
+# Standard imports
+import typing
+from typing import Optional
+import boto3
+import json
+from os import environ
+import urllib3
+from urllib.parse import urlunparse
+
+# Type hinting
+if typing.TYPE_CHECKING:
+    from mypy_boto3_secretsmanager import SecretsManagerClient
+    from mypy_boto3_ssm import SSMClient
+
+# Set globals
+ORCABUS_TOKEN_STR: Optional[str] = None
+HOSTNAME_STR: Optional[str] = None
+
+http = urllib3.PoolManager()
+
+LOCAL_HTTP_CACHE_PORT = 2773
+PARAMETER_URL = '/systemsmanager/parameters/get/'
+SECRETS_URL = '/secretsmanager/get/'
+
+def retrieve_extension_value(url, query):
+    url = str(urlunparse((
+        'http', f'localhost:{LOCAL_HTTP_CACHE_PORT}',
+        url, None,
+        "&".join(list(map(
+            lambda kv: f"{kv[0]}={kv[1]}",
+            query.items()
+        ))), None
+    )))
+    headers = {
+        "X-Aws-Parameters-Secrets-Token": environ.get('AWS_SESSION_TOKEN')
+    }
+    response = http.request("GET", url, headers=headers)
+    response = json.loads(response.data)
+    return response
+
+
+def get_ssm_value_from_cache(parameter_name: str) -> Optional[str]:
+    try:
+        return retrieve_extension_value(
+            PARAMETER_URL,
+            {
+                "name": parameter_name,
+            }
+        )['Parameter']['Value']
+    except Exception as e:
+        print("Got an exception while trying to get ssm value from cache")
+        print(e)
+        return None
+
+
+def get_secret_value_from_cache(secret_id: str) -> Optional[str]:
+    try:
+        return retrieve_extension_value(
+            SECRETS_URL,
+            {
+                "secretId": secret_id,
+            }
+        )['SecretString']
+    except Exception as e:
+        print("Got an exception while trying to get secret value from cache")
+        print(e)
+        return None
+
+def get_secretsmanager_client() -> 'SecretsManagerClient':
+    return boto3.client('secretsmanager')
+
+
+def get_ssm_client() -> 'SSMClient':
+    return boto3.client('ssm')
+
+
+def get_secret_value(secret_id) -> str:
+    """
+    Collect the secret value
+    :param secret_id:
+    :return:
+    """
+    secret_value_cached = get_secret_value_from_cache(secret_id)
+    if secret_value_cached is not None:
+        return secret_value_cached
+
+    # Get the boto3 response
+    get_secret_value_response = get_secretsmanager_client().get_secret_value(SecretId=secret_id)
+
+    return get_secret_value_response['SecretString']
+
+
+def get_ssm_value(parameter_name) -> str:
+    """
+    Collect the parameter from SSM
+    :param parameter_name:
+    :return:
+    """
+    ssm_parameter_cached = get_ssm_value_from_cache(parameter_name)
+    if ssm_parameter_cached is not None:
+        return ssm_parameter_cached
+
+    # Get the boto3 response
+    get_ssm_parameter_response = get_ssm_client().get_parameter(Name=parameter_name)
+
+    return get_ssm_parameter_response['Parameter']['Value']
+
+
+def get_orcabus_token() -> str:
+    """
+    From the AWS Secrets Manager, retrieve the OrcaBus token.
+    :return:
+    """
+    return (
+        json.loads(
+            get_secret_value(environ.get("ORCABUS_TOKEN_SECRET_ID"))
+        )['id_token']
+    )
+
+def get_hostname() -> str:
+    return get_ssm_value(environ.get("HOSTNAME_SSM_PARAMETER"))

package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/utils/miscell.py

@@ -0,0 +1,17 @@
+from urllib.parse import urlparse
+
+def get_bucket_key_pair_from_uri(s3_uri: str) -> (str, str):
+    """
+    Get the bucket and key from an s3 uri
+    :param s3_uri:
+    :return:
+    """
+    url_obj = urlparse(s3_uri)
+
+    s3_bucket = url_obj.netloc
+    s3_key = url_obj.path.lstrip('/')
+
+    if s3_bucket is None or s3_key is None:
+        raise ValueError(f"Invalid S3 URI: {s3_uri}")
+
+    return s3_bucket, s3_key

package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/utils/requests_helpers.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+from typing import Dict, Optional, List
+from urllib.parse import urlunparse
+
+# Standard imports
+import requests
+import logging
+from copy import deepcopy
+
+from requests import HTTPError
+
+# Locals
+from .aws_helpers import (
+    get_orcabus_token, get_hostname
+)
+
+# Globals
+DEFAULT_REQUEST_PARAMS = {
+    "rowsPerPage": 1000
+}
+
+# Set logging
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+
+def get_url(
+        endpoint: str,
+        subdomain: str
+) -> str:
+    """
+    Get the URL for the Metadata endpoint
+    :param endpoint:
+    :return:
+    """
+    # Get the hostname
+    hostname = get_hostname()
+
+    return str(urlunparse((
+        "https",
+        ".".join([subdomain, hostname]),
+        endpoint,
+        None, None, None
+    )))
+
+
+def get_request_response_results(url: str, params: Optional[Dict] = None) -> List[Dict]:
+    """
+    Run get response against the Metadata endpoint
+    :param url:
+    :param params:
+    :return:
+    """
+    # Get authorization header
+    headers = {
+        "Authorization": f"Bearer {get_orcabus_token()}"
+    }
+
+    req_params = deepcopy(DEFAULT_REQUEST_PARAMS)
+
+    req_params.update(
+        params if params is not None else {}
+    )
+
+
+    # Make the request
+    response = requests.get(
+        url,
+        headers=headers,
+        params=req_params
+    )
+
+    response.raise_for_status()
+
+    response_json = response.json()
+
+    if 'links' not in response_json.keys():
+        return [response_json]
+
+    if 'next' in response_json['links'].keys() and response_json['links']['next'] is not None:
+        return response_json['results'] + get_request_response_results(response_json['links']['next'])
+    return response_json['results']
+
+
+def get_request(url: str, params: Optional[Dict] = None) -> Dict:
+    # Get authorization header
+    headers = {
+        "Authorization": f"Bearer {get_orcabus_token()}"
+    }
+
+    # Make the request
+    response = requests.get(
+        url,
+        headers=headers,
+        params=params if params is not None else {}
+    )
+
+    try:
+        response.raise_for_status()
+    except HTTPError as e:
+        raise HTTPError(f"Error {e} - {response.text}") from e
+
+    return response.json()
+
+
+def patch_request(url: str, params: Optional[Dict] = None) -> Dict:
+    # Get authorization header
+    headers = {
+        "Authorization": f"Bearer {get_orcabus_token()}"
+    }
+
+    req_params = deepcopy(DEFAULT_REQUEST_PARAMS)
+
+    req_params.update(
+        params if params is not None else {}
+    )
+
+    # Make the request
+    response = requests.patch(
+        url,
+        headers=headers,
+        json=req_params
+    )
+
+    try:
+        response.raise_for_status()
+    except HTTPError as e:
+        raise HTTPError(f"Error {e} - {response.text}") from e
+
+    return response.json()
+
+
+def post_request(url: str, params: Optional[Dict] = None) -> Dict:
+    """
+    Run post request against the fastq endpoint
+    :param url:
+    :param params:
+    :return:
+    """
+    # Get authorization header
+    headers = {
+        "Authorization": f"Bearer {get_orcabus_token()}"
+    }
+
+    req_params = deepcopy(DEFAULT_REQUEST_PARAMS)
+
+    req_params.update(
+        params if params is not None else {}
+    )
+
+    # Make the request
+    response = requests.post(
+        url,
+        headers=headers,
+        json=req_params
+    )
+
+    try:
+        response.raise_for_status()
+    except HTTPError as e:
+        raise HTTPError(f"Error {e} - {response.text}") from e
+
+    return response.json()

package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/workflow/errors.py

@@ -0,0 +1,37 @@
+from typing import Optional
+
+
+class WorkflowRunNotFoundError(Exception):
+    def __init__(
+            self,
+            workflow_run_id: Optional[str] = None,
+            portal_run_id: Optional[str] = None,
+    ):
+        self.workflow_run_id = workflow_run_id
+        self.portal_run_id = portal_run_id
+        if workflow_run_id is not None:
+            self.message = f"Could not find workflow with orcabus run id '{workflow_run_id}'"
+        elif portal_run_id is not None:
+            self.message = f"Could not find workflow run portal run id '{portal_run_id}'"
+        else:
+            self.message = "Could not find workflow"
+        super().__init__(self.message)
+
+
+class WorkflowRunStateNotFoundError(Exception):
+    def __init__(
+            self,
+            workflow_run_id: Optional[str] = None,
+            portal_run_id: Optional[str] = None,
+            status: Optional[str] = None,
+    ):
+        self.workflow_run_id = workflow_run_id
+        self.portal_run_id = portal_run_id
+        self.status = status
+        if workflow_run_id is not None:
+            self.message = f"Could not find workflow with orcabus run id '{workflow_run_id}', status '{status}'"
+        elif portal_run_id is not None:
+            self.message = f"Could not find workflow run portal run id '{portal_run_id}', status '{status}'"
+        else:
+            self.message = "Could not find workflow"
+        super().__init__(self.message)

package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/workflow/metadata_helpers.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+
+"""
+Get workflows from library id
+"""
+
+# Standard imports
+from typing import List
+
+
+# Local imports
+from .requests_helpers import get_request_response_results
+from .globals import WORKFLOW_RUN_ENDPOINT
+from .models import WorkflowRunDetail
+
+
+def get_workflows_from_library_id(library_id: str) -> List[WorkflowRunDetail]:
+    """
+    Use the query libraries__libraryId to get workflows from a library id
+    :param library_id:
+    :return:
+    """
+    return get_request_response_results(
+        WORKFLOW_RUN_ENDPOINT,
+        params={
+            "libraries__libraryId": library_id
+        }
+    )