@orcabus/platform-cdk-constructs 0.0.6 → 0.0.7-alpha.4

package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/models.py

@@ -0,0 +1,253 @@
+#!/usr/bin/env python3
+
+"""
+{
+  "id": "fqr.01JJY7P1AVFGHGVMEDE8T4VWJG",
+  "rgid": "ATCCACTG+ACGCACCT.2",
+  "index": "ATCCACTG",
+  "index2": "ACGCACCT",
+  "lane": 2,
+  "instrumentRunId": "230223_A00130_0244_AHN3W3DSX5",
+  "library": {
+    "orcabusId": "lib.01JBMVFP45C2EZRVK67P8JY1D2",
+    "libraryId": "L2300223"
+  },
+  "readSet": {
+    "r1": {
+      "s3IngestId": "019387bd-2494-7c00-9e41-03e8b6a73306",
+      "gzipCompressionSizeInBytes": 49532847794,
+      "rawMd5sum": "19e339fdb3c42f0133f5f3b1f9d188e0",  // pragma: allowlist secret
+      "s3Uri": "s3://archive-prod-fastq-503977275616-ap-southeast-2/v1/year=2023/month=02/230223_A00130_0244_AHN3W3DSX5/202411226f4f7af0/WGS_TsqNano/MDX230039_L2300223_S7_L002_R1_001.fastq.ora"
+    },
+    "r2": {
+      "s3IngestId": "019387bd-9177-79c1-a489-d940ecc11b11",
+      "gzipCompressionSizeInBytes": 53189277581,
+      "rawMd5sum": "e857de35a8ca008589d24b2e0f647cc7", // pragma: allowlist secret
+      "s3Uri": "s3://archive-prod-fastq-503977275616-ap-southeast-2/v1/year=2023/month=02/230223_A00130_0244_AHN3W3DSX5/202411226f4f7af0/WGS_TsqNano/MDX230039_L2300223_S7_L002_R2_001.fastq.ora"
+    },
+    "compressionFormat": "ORA"
+  },
+  "qc": null,
+  "readCount": null,
+  "baseCountEst": null,
+  "isValid": true,
+  "ntsm": null
+}
+"""
+
+from enum import Enum
+from typing import (
+    TypedDict,
+    Optional,
+    Dict,
+    List, NotRequired, Union, Literal
+)
+from datetime import datetime
+
+
+class JobType(Enum):
+    QC = 'QC'
+    FILE_COMPRESSION = 'FILE_COMPRESSION'
+    NTSM = 'NTSM'
+
+
+class JobStatus(Enum):
+    PENDING = "PENDING"
+    RUNNING = "RUNNING"
+    FAILED = "FAILED"
+    SUCCEEDED = "SUCCEEDED"
+
+
+class FileStorageObject(TypedDict):
+    s3IngestId: str
+    s3Uri: str
+    storageClass: str
+    sha256: str
+
+
+class FastqStorageObject(FileStorageObject):
+    gzipCompressionSizeInBytes: int
+    rawMd5sum: str
+
+
+class ReadSet(TypedDict):
+    r1: FastqStorageObject
+    r2: FastqStorageObject
+    compressionFormat: str
+
+
+class Library(TypedDict):
+    orcabusId: str
+    libraryId: str
+
+
+class FastqListRowCreate(TypedDict):
+    fastqSetId: Optional[str]
+    index: str
+    lane: int
+    instrumentRunId: str
+    library: Library
+    platform: Optional[str]
+    center: Optional[str]
+    date: Optional[datetime]
+    readSet: Optional[ReadSet]
+    qc: Optional[Dict]
+    readCount: Optional[int]
+    baseCountEst: Optional[int]
+    isValid: Optional[bool]
+    ntsm: Optional[FileStorageObject]
+
+
+class FastqListRow(TypedDict):
+    id: str
+    fastqSetId: Optional[str]
+    index: str
+    lane: int
+    instrumentRunId: str
+    library: Library
+    platform: Optional[str]
+    center: Optional[str]
+    date: Optional[datetime]
+    readSet: Optional[ReadSet]
+    qc: Optional[Dict]
+    readCount: Optional[int]
+    baseCountEst: Optional[int]
+    isValid: Optional[bool]
+    ntsm: Optional[FileStorageObject]
+
+
+class FastqSetCreate(TypedDict):
+    library: Library
+    fastqSet: List[Union[str, FastqListRow]]
+    allowAdditionalFastq: bool
+    isCurrentFastqSet: bool
+
+
+class FastqSet(TypedDict):
+    id: str
+    library: Library
+    fastqSet: List[FastqListRow]
+    allowAdditionalFastq: bool
+    isCurrentFastqSet: bool
+
+
+class QcStats(TypedDict):
+    insertSizeEstimate: int
+    rawWgsCoverageEstimate: int
+    r1Q20Fraction: float
+    r2Q20Fraction: float
+    r1GcFraction: float
+    r2GcFraction: float
+
+
+class ReadCount(TypedDict):
+    readCount: int
+    baseCountEst: int
+
+
+class FileCompressionInformation(TypedDict):
+    compressionFormat: str
+    r1GzipCompressionSizeInBytes: Optional[int]
+    r2GzipCompressionSizeInBytes: Optional[int]
+    r1RawMd5sum: Optional[int]
+    r2RawMd5sum: Optional[int]
+
+
+class FastqListRowDict(TypedDict):
+    rgid: str
+    rglb: str
+    rgsm: str
+    lane: int
+    rgcn: str
+    rgds: str
+    rgdt: str
+    rgpl: str
+    read1FileUri: str
+    read2FileUri: str
+
+
+
+class Job(TypedDict):
+    id: str
+    fastqId: str
+    jobType: JobType
+    stepsExecutionArn: str
+    status: JobStatus
+    startTime: datetime
+    endTime: Optional[datetime]
+
+
+class BoolAllEnum(Enum):
+    ALL = "ALL"
+    true = True
+    false = False
+
+
+class FastqGetResponseParameters(TypedDict):
+    includeS3Details: NotRequired[BoolAllEnum]
+
+
+class StandardQueryParameters(TypedDict):
+    page: NotRequired[int]
+    rowsPerPage: NotRequired[int]
+
+
+class MetadataQueryParameter(TypedDict):
+    library: NotRequired[str]
+    sample: NotRequired[str]
+    subject: NotRequired[str]
+    individual: NotRequired[str]
+    project: NotRequired[str]
+
+
+MetadataQueryParametersList = TypedDict(
+    'MetadataQueryParametersList',
+    {
+        'library[]': NotRequired[List[str]],
+        'sample[]': NotRequired[List[str]],
+        'subject[]': NotRequired[List[str]],
+        'individual[]': NotRequired[List[str]],
+        'project[]': NotRequired[List[str]],
+    }
+)
+
+class InstrumentRunIdQueryParameters(TypedDict):
+    index: NotRequired[str]
+    lane: NotRequired[int]
+    instrumentRunId: NotRequired[str]
+
+InstrumentRunIdQueryParametersList = TypedDict(
+    'InstrumentRunIdQueryParametersList',
+    {
+        'instrumentRunId[]': NotRequired[List[str]],
+    }
+)
+
+
+class FastqListRowQueryParameters(
+    StandardQueryParameters,
+    MetadataQueryParameter,
+    InstrumentRunIdQueryParameters,
+    MetadataQueryParametersList,
+    InstrumentRunIdQueryParametersList
+):
+    valid: NotRequired[BoolAllEnum]
+    includeS3Details: NotRequired[BoolAllEnum]
+    fastqSetId: NotRequired[str]
+
+
+class FastqSetQueryParameters(
+    StandardQueryParameters,
+    MetadataQueryParameter,
+    MetadataQueryParametersList,
+    InstrumentRunIdQueryParameters
+):
+    currentFastqSet: NotRequired[BoolAllEnum]
+    allowAdditionalFastq: NotRequired[BoolAllEnum]
+    includeS3Details: NotRequired[BoolAllEnum]
+
+# Additional types
+VALID_BATCH_KEYS = Literal[
+    'library', 'sample', 'subject',
+    'individual', 'project', 'instrumentRunId'
+]

package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/query_helpers.py

@@ -0,0 +1,248 @@
+#!/usr/bin/env python3
+
+"""
+Query helpers -
+
+get_fastqs_in_instrument_run_id
+
+get_fastqs_in_library
+
+get_fastqs_in_sample
+
+get_fastqs_in_subject
+
+get_fastqs_in_individual
+
+get_fastqs_in_project
+
+get_fastq_by_rgid_and_instrument_run_id
+
+"""
+from functools import reduce
+from itertools import batched
+from operator import concat
+from typing import List, Unpack
+
+from . import get_fastq_request_response_results, get_fastq_request
+from .globals import FASTQ_LIST_ROW_ENDPOINT, FASTQ_SET_ENDPOINT
+from .models import FastqListRow, FastqSet, Job, FastqListRowQueryParameters, FastqSetQueryParameters, \
+    FastqGetResponseParameters, VALID_BATCH_KEYS
+
+
+def get_fastq(fastq_id: str, **kwargs: Unpack[FastqGetResponseParameters]) -> FastqListRow:
+    # Raise error if any of the kwargs are not in the FastqSetQueryParameters
+    for key in kwargs.keys():
+        if key not in FastqGetResponseParameters.__annotations__:
+            raise ValueError(f"Invalid parameter: {key}")
+
+    return FastqListRow(
+        **get_fastq_request(
+            f"{FASTQ_LIST_ROW_ENDPOINT}/{fastq_id}",
+            params=dict(kwargs)
+        )
+    )
+
+
+def get_fastq_set(
+        fastq_set_id: str,
+        **kwargs: Unpack[FastqGetResponseParameters]
+) -> FastqSet:
+    """
+    Get the fastq set by id
+    :param fastq_set_id:
+    :param kwargs:
+    :return:
+    """
+    # Raise error if any of the kwargs are not in the FastqSetQueryParameters
+    for key in kwargs.keys():
+        if key not in FastqSetQueryParameters.__annotations__:
+            raise ValueError(f"Invalid parameter: {key}")
+
+    return FastqSet(
+        **get_fastq_request(
+            f"{FASTQ_SET_ENDPOINT}/{fastq_set_id}",
+            params=dict(kwargs)
+        )
+    )
+
+
+def get_fastqs(**kwargs: Unpack[FastqListRowQueryParameters]) -> List[FastqListRow]:
+    """
+    Get all fastqs
+    """
+    # Raise error if any of the kwargs are not in the FastqListRowQueryParameters
+    for key in kwargs.keys():
+        if key not in FastqListRowQueryParameters.__annotations__:
+            raise ValueError(f"Invalid parameter: {key}")
+
+    return get_fastq_request_response_results(
+        FASTQ_LIST_ROW_ENDPOINT,
+        params=dict(kwargs)
+    )
+
+
+def get_fastq_sets(**kwargs: Unpack[FastqSetQueryParameters]) -> List[FastqSet]:
+    """
+    Get the fastq set
+    :param args:
+    :param kwargs:
+    :return:
+    """
+    # Raise error if any of the kwargs are not in the FastqListRowQueryParameters
+    for key in kwargs.keys():
+        if key not in FastqSetQueryParameters.__annotations__:
+            raise ValueError(f"Invalid parameter: {key}")
+
+    return get_fastq_request_response_results(
+        FASTQ_SET_ENDPOINT,
+        params=dict(kwargs)
+    )
+
+
+def get_fastqs_in_instrument_run_id(instrument_run_id: str):
+    """
+    Get all fastqs in an instrument run id
+    """
+    return get_fastqs(
+        instrumentRunId=instrument_run_id
+    )
+
+
+def get_fastqs_in_library(library_id: str):
+    """
+    Get all fastqs in a library
+    """
+    return get_fastqs(
+        library=library_id,
+    )
+
+
+def get_fastqs_batched(
+        item_key: VALID_BATCH_KEYS,
+        item_list: List[str],
+        batch_size: int = 100,
+        **kwargs
+) -> List[FastqListRow]:
+    """
+    Get all fastqs in a list of libraries
+    """
+    # Split by groups of 50
+    item_lists = batched(item_list, batch_size)
+
+    # Get the s3 objects
+    try:
+        return list(reduce(
+            concat,
+            list(map(
+                lambda item_batch_:
+                get_fastqs(
+                    **{
+                        f"{item_key}[]": list(item_batch_),
+                    },
+                    **kwargs,
+                    rowsPerPage=100
+                ),
+                item_lists
+            ))
+        ))
+    except TypeError as e:
+        # TypeError: reduce() of empty iterable with no initial value
+        return []
+
+
+def get_fastqs_in_library_list(
+        library_id_list: List[str]
+) -> List[FastqListRow]:
+    """
+    Get all fastqs in a list of libraries
+    """
+    return get_fastqs_batched(
+        item_key="library",
+        item_list=library_id_list,
+        batch_size=50
+    )
+
+
+def get_fastqs_in_libraries_and_instrument_run_id(library_id_list, instrument_run_id):
+    """
+    Get all fastqs in a list of libraries and instrument run id
+    :param library_id_list:
+    :param instrument_run_id:
+    :return:
+    """
+    return get_fastqs(
+        instrumentRunId=instrument_run_id,
+        **{
+            "library[]": library_id_list,
+        }
+    )
+
+
+def get_fastqs_in_sample(sample_id: str):
+    """
+    Get all fastqs in a sample
+    """
+    return get_fastqs(
+        sample=sample_id
+    )
+
+
+def get_fastqs_in_sample_list(sample_id_list: List[str]):
+    """
+    Get all fastqs in a list of samples
+    :param sample_id_list:
+    :return:
+    """
+    return get_fastqs_batched(
+        item_key="sample",
+        item_list=sample_id_list,
+        batch_size=50
+    )
+
+
+def get_fastqs_in_subject(subject_id):
+    """
+    Get all fastqs in a subject
+    """
+    return get_fastqs(
+        subject=subject_id
+    )
+
+
+def get_fastqs_in_individual(individual_id):
+    """
+    Get all fastqs in an individual
+    """
+    return get_fastqs(
+        individual=individual_id
+    )
+
+
+def get_fastqs_in_project(project_id):
+    """
+    Get all fastqs in a project
+    """
+    return get_fastqs(
+        project=project_id
+    )
+
+
+def get_fastq_list_rows_in_fastq_set(fastq_set_id):
+    """
+    Get all fastqs in a fastq set
+    """
+    return get_fastqs(
+        fastqSetId=fastq_set_id
+    )
+
+
+def get_fastq_jobs(fastq_id: str) -> List[Job]:
+    """
+    Get all fastqs in a fastq set
+    """
+    return list(map(
+        lambda job_iter_: Job(**job_iter_),
+        get_fastq_request_response_results(
+            f"{FASTQ_LIST_ROW_ENDPOINT}/{fastq_id}/jobs"
+        )
+    ))

package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/update_helpers.py

@@ -0,0 +1,221 @@
+#!/usr/bin/env python3
+
+"""
+Update helpers for the update script.
+
+- add_qc_stats
+- add_read_count
+- add_ntsm_storage_object / add_ntsm
+- add_fastq_pair_storage_object / add_read_set
+- detach_fastq_pair_storage_object / detach_read_set
+- validate
+- invalidate
+"""
+# Standard imports
+
+# Local imports
+from . import fastq_patch_request
+from .globals import FASTQ_LIST_ROW_ENDPOINT, FASTQ_SET_ENDPOINT
+from .models import (
+    QcStats, FastqListRow, ReadCount,
+    FileCompressionInformation, FileStorageObject, ReadSet,
+    FastqSet
+)
+
+
+def add_qc_stats(fastq_id: str, qc_stats: QcStats) -> FastqListRow:
+    """
+    Add QC stats to a fastq_id.
+
+    :param fastq_id: Fastq str
+    :param qc_stats: Dictionary of QC stats
+    """
+    # Raise error if any of the kwargs are not in the QcStats
+    for key in qc_stats.keys():
+        if key not in QcStats.__annotations__:
+            raise ValueError(f"Invalid parameter: {key}")
+
+    return fastq_patch_request(
+        f"{FASTQ_LIST_ROW_ENDPOINT}/{fastq_id}/addQcStats",
+        params=dict(qc_stats)
+    )
+
+
+def add_read_count(fastq_id: str, read_count: ReadCount) -> FastqListRow:
+    """
+    Add read count to a fastq id
+    :param fastq_id:
+    :param read_count:
+    :return:
+    """
+    for key in read_count.keys():
+        if key not in ReadCount.__annotations__:
+            raise ValueError(f"Invalid parameter: {key}")
+
+    return fastq_patch_request(
+        f"{FASTQ_LIST_ROW_ENDPOINT}/{fastq_id}/addReadCount",
+        params=dict(read_count)
+    )
+
+
+def add_file_compression_information(fastq_id: str, file_compression_information: FileCompressionInformation) -> FastqListRow:
+    """
+    Add file compression information to a fastq id
+    :param fastq_id:
+    :param file_compression_information:
+    :return:
+    """
+    for key in file_compression_information.keys():
+        if key not in FileCompressionInformation.__annotations__:
+            raise ValueError(f"Invalid parameter: {key}")
+
+    return fastq_patch_request(
+        f"{FASTQ_LIST_ROW_ENDPOINT}/{fastq_id}/addFileCompressionInformation",
+        params=dict(file_compression_information)
+    )
+
+
+def add_ntsm_storage_object(fastq_id: str, ntsm_fastq_storage_object: FileStorageObject) -> FastqListRow:
+    """
+    Add a Ntsm storage object to a fastq id.
+
+    :param fastq_id: Fastq str
+    :param ntsm_fastq_storage_object: Ntsm str
+    """
+    for key in ntsm_fastq_storage_object.keys():
+        if key not in FileStorageObject.__annotations__:
+            raise ValueError(f"Invalid parameter: {key}")
+
+    return fastq_patch_request(
+        f"{FASTQ_LIST_ROW_ENDPOINT}/{fastq_id}/addNtsmStorageObject",
+        params=dict(ntsm_fastq_storage_object)
+    )
+
+
+def add_read_set(fastq_id: str, read_set: ReadSet) -> FastqListRow:
+    """
+    Add a read set to a fastq id.
+
+    :param fastq_id: Fastq str
+    :param read_set: ReadSet str
+    """
+    for key in read_set.keys():
+        if key not in ReadSet.__annotations__:
+            raise ValueError(f"Invalid parameter: {key}")
+
+    return fastq_patch_request(
+        f"{FASTQ_LIST_ROW_ENDPOINT}/{fastq_id}/addFastqPairStorageObject",
+        params=dict(read_set)
+    )
+
+
+def detach_read_set(fastq_id: str, read_set: ReadSet) -> FastqListRow:
+    """
+    Detach a read set to a fastq id.
+
+    :param fastq_id: Fastq str
+    :param read_set: ReadSet str
+    """
+    for key in read_set.keys():
+        if key not in ReadSet.__annotations__:
+            raise ValueError(f"Invalid parameter: {key}")
+
+    return fastq_patch_request(
+        f"{FASTQ_LIST_ROW_ENDPOINT}/{fastq_id}/detachFastqPairStorageObject",
+        params=dict(read_set)
+    )
+
+
+def validate_fastq(fastq_id: str) -> FastqListRow:
+    """
+    Validate a fastq id.
+
+    :param fastq_id: Fastq str
+    """
+    return fastq_patch_request(
+        f"{FASTQ_LIST_ROW_ENDPOINT}/{fastq_id}/validate"
+    )
+
+
+def invalidate_fastq(fastq_id: str) -> FastqListRow:
+    """
+    Invalidate a fastq id.
+
+    :param fastq_id: Fastq str
+    """
+    return fastq_patch_request(
+        f"{FASTQ_LIST_ROW_ENDPOINT}/{fastq_id}/invalidate"
+    )
+
+
+def link_fastq_list_row_to_fastq_set(fastq_id: str, fastq_set_id: str) -> FastqSet:
+    """
+    Link a fastq id to a fastq set.
+
+    :param fastq_id:
+    :param fastq_set_id:
+    :return:
+    """
+    return fastq_patch_request(
+        f"{FASTQ_SET_ENDPOINT}/{fastq_set_id}/linkFastq/{fastq_id}"
+    )
+
+
+def unlink_fastq_list_row_from_fastq_set(fastq_id: str, fastq_set_id: str) -> FastqSet:
+    """
+    Unlink a fastq id from a fastq set.
+
+    :param fastq_id:
+    :param fastq_set_id:
+    :return:
+    """
+
+    return fastq_patch_request(
+        f"{FASTQ_SET_ENDPOINT}/{fastq_set_id}/unlinkFastq/{fastq_id}"
+    )
+
+
+def allow_additional_fastqs_to_fastq_set(fastq_set_id: str) -> FastqSet:
+    """
+    Allow additional fastqs to be added to a fastq set.
+    :param fastq_set_id:
+    :return:
+    """
+
+    return fastq_patch_request(
+        f"{FASTQ_SET_ENDPOINT}/{fastq_set_id}/allowAdditionalFastqs"
+    )
+
+
+def disallow_additional_fastqs_to_fastq_set(fastq_set_id: str) -> FastqSet:
+    """
+    Disallow additional fastqs to be added to a fastq set.
+    :param fastq_set_id:
+    :return:
+    """
+
+    return fastq_patch_request(
+        f"{FASTQ_SET_ENDPOINT}/{fastq_set_id}/disallowAdditionalFastqs"
+    )
+
+
+def set_is_current_fastq_set(fastq_set_id: str) -> FastqSet:
+    """
+    Set a fastq set as current.
+    :param fastq_set_id:
+    :return:
+    """
+    return fastq_patch_request(
+        f"{FASTQ_SET_ENDPOINT}/{fastq_set_id}/currentFastqSet"
+    )
+
+
+def set_is_not_current_fastq_set(fastq_set_id: str) -> FastqSet:
+    """
+    Set a fastq set as not current.
+    :param fastq_set_id:
+    :return:
+    """
+    return fastq_patch_request(
+        f"{FASTQ_SET_ENDPOINT}/{fastq_set_id}/notCurrentFastqSet"
+    )