@orcabus/platform-cdk-constructs 0.0.33 → 0.0.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.jsii +29 -29
- package/api-gateway/api-gateway.js +1 -1
- package/deployment-stack-pipeline/artifact-bucket.js +1 -1
- package/deployment-stack-pipeline/pipeline.js +36 -3
- package/dynamodb/index.js +2 -2
- package/ecs/index.js +1 -1
- package/lambda/index.js +1 -1
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/data_sharing/__init__.py +70 -0
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/data_sharing/globals.py +15 -0
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/data_sharing/models.py +28 -0
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/data_sharing/update_helpers.py +53 -0
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/__init__.py +12 -2
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/create_helpers.py +22 -5
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/globals.py +1 -1
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/job_helpers.py +16 -4
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/models.py +81 -20
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/query_helpers.py +25 -14
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/update_helpers.py +42 -18
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/workflow_helpers.py +3 -2
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq_unarchiving/__init__.py +4 -1
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq_unarchiving/create_helpers.py +2 -2
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq_unarchiving/globals.py +0 -8
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq_unarchiving/models.py +13 -14
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq_unarchiving/query_helpers.py +3 -3
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq_unarchiving/update_helpers.py +3 -3
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/filemanager/__init__.py +2 -0
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/filemanager/file_helpers.py +18 -11
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/filemanager/globals.py +0 -28
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/filemanager/models.py +19 -2
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/utils/requests_helpers.py +9 -2
- package/monitored-queue/index.js +1 -1
- package/named-lambda-role/index.js +1 -1
- package/package.json +1 -1
- package/provider-function/index.js +1 -1
- package/shared-config/networking.js +2 -2
- package/shared-config/slack.js +1 -1
|
@@ -13,28 +13,27 @@
|
|
|
13
13
|
|
|
14
14
|
from typing import (
|
|
15
15
|
TypedDict, NotRequired,
|
|
16
|
+
Literal
|
|
16
17
|
)
|
|
17
18
|
|
|
18
|
-
|
|
19
|
+
JobType = Literal[
|
|
20
|
+
"S3_UNARCHIVING",
|
|
21
|
+
]
|
|
19
22
|
|
|
20
23
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
FAILED = "FAILED"
|
|
29
|
-
ABORTED = "ABORTED"
|
|
30
|
-
SUCCEEDED = "SUCCEEDED"
|
|
31
|
-
|
|
24
|
+
JobStatusType = Literal[
|
|
25
|
+
"PENDING",
|
|
26
|
+
"RUNNING",
|
|
27
|
+
"FAILED",
|
|
28
|
+
"ABORTED",
|
|
29
|
+
"SUCCEEDED",
|
|
30
|
+
]
|
|
32
31
|
|
|
33
32
|
class Job(TypedDict):
|
|
34
33
|
id: str
|
|
35
34
|
jobType: JobType
|
|
36
35
|
stepsExecutionArn: str
|
|
37
|
-
status:
|
|
36
|
+
status: JobStatusType
|
|
38
37
|
startTime: str
|
|
39
38
|
endTime: str
|
|
40
39
|
errorMessages: str
|
|
@@ -42,7 +41,7 @@ class Job(TypedDict):
|
|
|
42
41
|
|
|
43
42
|
class JobQueryParameters(TypedDict):
|
|
44
43
|
fastqId: NotRequired[str]
|
|
45
|
-
status: NotRequired[
|
|
44
|
+
status: NotRequired[JobStatusType]
|
|
46
45
|
createdAfter: NotRequired[str]
|
|
47
46
|
createdBefore: NotRequired[str]
|
|
48
47
|
completedAfter: NotRequired[str]
|
package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq_unarchiving/query_helpers.py
CHANGED
|
@@ -14,7 +14,7 @@ from typing import List, Unpack
|
|
|
14
14
|
|
|
15
15
|
# Local imports
|
|
16
16
|
from . import get_fastq_unarchiving_request_response_results
|
|
17
|
-
from .models import Job,
|
|
17
|
+
from .models import Job, JobStatusType, JobQueryParameters
|
|
18
18
|
|
|
19
19
|
from .globals import JOB_ENDPOINT
|
|
20
20
|
|
|
@@ -40,7 +40,7 @@ def get_unarchiving_job_list(**kwargs: Unpack[JobQueryParameters]) -> List[Job]:
|
|
|
40
40
|
|
|
41
41
|
def get_job_list_for_fastq(
|
|
42
42
|
fastq_id: str,
|
|
43
|
-
job_status:
|
|
43
|
+
job_status: JobStatusType
|
|
44
44
|
) -> List[Job]:
|
|
45
45
|
"""
|
|
46
46
|
Check if fastq in job list
|
|
@@ -48,5 +48,5 @@ def get_job_list_for_fastq(
|
|
|
48
48
|
"""
|
|
49
49
|
return get_unarchiving_job_list(
|
|
50
50
|
fastqId=fastq_id,
|
|
51
|
-
status=job_status
|
|
51
|
+
status=job_status
|
|
52
52
|
)
|
package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq_unarchiving/update_helpers.py
CHANGED
|
@@ -17,13 +17,13 @@ from typing import Optional
|
|
|
17
17
|
|
|
18
18
|
# Local imports
|
|
19
19
|
from . import fastq_unarchiving_patch_request
|
|
20
|
-
from .globals import
|
|
21
|
-
from .models import Job
|
|
20
|
+
from .globals import JOB_ENDPOINT
|
|
21
|
+
from .models import Job, JobStatusType
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
def update_status(
|
|
25
25
|
job_id: str,
|
|
26
|
-
job_status:
|
|
26
|
+
job_status: JobStatusType,
|
|
27
27
|
error_message: Optional[str] = None
|
|
28
28
|
) -> Job:
|
|
29
29
|
"""
|
|
@@ -40,10 +40,12 @@ def get_file_manager_request(
|
|
|
40
40
|
|
|
41
41
|
def file_manager_patch_request(
|
|
42
42
|
endpoint: str,
|
|
43
|
+
json_data: Optional[Dict] = None,
|
|
43
44
|
params: Optional[Dict] = None
|
|
44
45
|
):
|
|
45
46
|
return patch_request(
|
|
46
47
|
get_file_manager_url(endpoint),
|
|
48
|
+
json_data=json_data,
|
|
47
49
|
params=params
|
|
48
50
|
)
|
|
49
51
|
|
|
@@ -1,24 +1,30 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
|
+
|
|
3
|
+
# Standard imports
|
|
4
|
+
import json
|
|
2
5
|
from functools import reduce
|
|
3
6
|
from operator import concat
|
|
4
7
|
from typing import List, Dict, Union
|
|
5
8
|
import typing
|
|
6
|
-
|
|
7
9
|
import boto3
|
|
10
|
+
from datetime import datetime, timedelta, timezone
|
|
11
|
+
from urllib.parse import urlparse
|
|
12
|
+
from itertools import batched
|
|
8
13
|
|
|
14
|
+
# Local imports
|
|
9
15
|
from .errors import S3FileNotFoundError, S3DuplicateFileCopyError
|
|
10
|
-
from .models import FileObject
|
|
16
|
+
from .models import FileObject, StorageClassPriority
|
|
11
17
|
from ..utils.miscell import get_bucket_key_pair_from_uri
|
|
12
|
-
from . import
|
|
18
|
+
from . import (
|
|
19
|
+
get_file_manager_request_response_results,
|
|
20
|
+
get_file_manager_request,
|
|
21
|
+
file_manager_patch_request
|
|
22
|
+
)
|
|
13
23
|
from .globals import (
|
|
14
24
|
S3_LIST_ENDPOINT,
|
|
15
25
|
S3_BUCKETS_BY_ACCOUNT_ID,
|
|
16
|
-
S3_PREFIXES_BY_ACCOUNT_ID
|
|
17
|
-
StorageEnum, StoragePriority,
|
|
26
|
+
S3_PREFIXES_BY_ACCOUNT_ID
|
|
18
27
|
)
|
|
19
|
-
from datetime import datetime, timedelta, timezone
|
|
20
|
-
from urllib.parse import urlparse
|
|
21
|
-
from itertools import batched
|
|
22
28
|
|
|
23
29
|
if typing.TYPE_CHECKING:
|
|
24
30
|
from mypy_boto3_sts import STSClient
|
|
@@ -94,7 +100,7 @@ def get_file_object_from_ingest_id(ingest_id: str, **kwargs) -> FileObject:
|
|
|
94
100
|
))
|
|
95
101
|
|
|
96
102
|
# Order by storage class
|
|
97
|
-
file_objects_list.sort(key=lambda file_obj_iter_:
|
|
103
|
+
file_objects_list.sort(key=lambda file_obj_iter_: StorageClassPriority[file_obj_iter_['storageClass']])
|
|
98
104
|
|
|
99
105
|
# Return as a FileObject model
|
|
100
106
|
return file_objects_list[0]
|
|
@@ -263,8 +269,9 @@ def get_s3_objs_from_ingest_ids_map(ingest_ids: List[str], **kwargs) -> List[Dic
|
|
|
263
269
|
continue
|
|
264
270
|
|
|
265
271
|
s3_objects_match.sort(
|
|
266
|
-
key=lambda s3_object_iter_:
|
|
267
|
-
|
|
272
|
+
key=lambda s3_object_iter_: StorageClassPriority[
|
|
273
|
+
s3_object_iter_['fileObject']['storageClass']
|
|
274
|
+
]
|
|
268
275
|
)
|
|
269
276
|
|
|
270
277
|
s3_objects_by_ingest_id_filtered.append(
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
#!/usr/bin/env python
|
|
2
|
-
from enum import Enum
|
|
3
2
|
|
|
4
3
|
# AWS PARAMETERS
|
|
5
4
|
FILEMANAGER_SUBDOMAIN_NAME = "file"
|
|
@@ -41,30 +40,3 @@ S3_PREFIXES_BY_ACCOUNT_ID = {
|
|
|
41
40
|
},
|
|
42
41
|
}
|
|
43
42
|
|
|
44
|
-
# FROM FileManager Schema
|
|
45
|
-
# "DeepArchive"
|
|
46
|
-
# "Glacier"
|
|
47
|
-
# "GlacierIr"
|
|
48
|
-
# "IntelligentTiering"
|
|
49
|
-
# "OnezoneIa"
|
|
50
|
-
# "Outposts"
|
|
51
|
-
# "ReducedRedundancy"
|
|
52
|
-
# "Snow"
|
|
53
|
-
# "Standard"
|
|
54
|
-
# "StandardIa"
|
|
55
|
-
class StorageEnum(Enum):
|
|
56
|
-
STANDARD = "Standard"
|
|
57
|
-
STANDARD_IA = "StandardIa"
|
|
58
|
-
INTELLIGENT_TIERING = "IntelligentTiering"
|
|
59
|
-
GLACIER_INSTANT_RETRIEVAL = "GlacierIr"
|
|
60
|
-
GLACIER = "Glacier"
|
|
61
|
-
DEEP_ARCHIVE = "DeepArchive"
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
class StoragePriority(Enum):
|
|
65
|
-
STANDARD = 1
|
|
66
|
-
STANDARD_IA = 2
|
|
67
|
-
INTELLIGENT_TIERING = 3
|
|
68
|
-
GLACIER_INSTANT_RETRIEVAL = 4
|
|
69
|
-
GLACIER = 5
|
|
70
|
-
DEEP_ARCHIVE = 6
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import Optional, TypedDict, Dict
|
|
1
|
+
from typing import Optional, TypedDict, Dict, Literal
|
|
2
2
|
|
|
3
3
|
"""
|
|
4
4
|
Example File Object response
|
|
@@ -26,6 +26,23 @@ Example File Object response
|
|
|
26
26
|
}
|
|
27
27
|
"""
|
|
28
28
|
|
|
29
|
+
StorageClassType = Literal[
|
|
30
|
+
"Standard",
|
|
31
|
+
"StandardIa",
|
|
32
|
+
"IntelligentTiering",
|
|
33
|
+
"GlacierIr",
|
|
34
|
+
"Glacier",
|
|
35
|
+
"DeepArchive",
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
StorageClassPriority: Dict[StorageClassType, int] = {
|
|
39
|
+
"Standard": 1,
|
|
40
|
+
"StandardIa": 2,
|
|
41
|
+
"IntelligentTiering": 3,
|
|
42
|
+
"GlacierIr": 4,
|
|
43
|
+
"Glacier": 5,
|
|
44
|
+
"DeepArchive": 6,
|
|
45
|
+
}
|
|
29
46
|
|
|
30
47
|
class FileObject(TypedDict):
|
|
31
48
|
# Identifier
|
|
@@ -47,7 +64,7 @@ class FileObject(TypedDict):
|
|
|
47
64
|
numberReordered: int
|
|
48
65
|
sequencer: str
|
|
49
66
|
size: int
|
|
50
|
-
storageClass:
|
|
67
|
+
storageClass: StorageClassType
|
|
51
68
|
|
|
52
69
|
# Attribute attributes
|
|
53
70
|
attributes: Optional[Dict]
|
|
@@ -9,6 +9,8 @@ from copy import deepcopy
|
|
|
9
9
|
|
|
10
10
|
from requests import HTTPError
|
|
11
11
|
|
|
12
|
+
from fastapi.encoders import jsonable_encoder
|
|
13
|
+
|
|
12
14
|
# Locals
|
|
13
15
|
from .aws_helpers import (
|
|
14
16
|
get_orcabus_token, get_hostname
|
|
@@ -103,7 +105,11 @@ def get_request(url: str, params: Optional[Dict] = None) -> Dict:
|
|
|
103
105
|
return response.json()
|
|
104
106
|
|
|
105
107
|
|
|
106
|
-
def patch_request(
|
|
108
|
+
def patch_request(
|
|
109
|
+
url: str,
|
|
110
|
+
json_data: Optional[Dict] = None,
|
|
111
|
+
params: Optional[Dict] = None
|
|
112
|
+
) -> Dict:
|
|
107
113
|
# Get authorization header
|
|
108
114
|
headers = {
|
|
109
115
|
"Authorization": f"Bearer {get_orcabus_token()}"
|
|
@@ -119,7 +125,8 @@ def patch_request(url: str, params: Optional[Dict] = None) -> Dict:
|
|
|
119
125
|
response = requests.patch(
|
|
120
126
|
url,
|
|
121
127
|
headers=headers,
|
|
122
|
-
|
|
128
|
+
params=req_params,
|
|
129
|
+
json=json_data
|
|
123
130
|
)
|
|
124
131
|
|
|
125
132
|
try:
|
package/monitored-queue/index.js
CHANGED
|
@@ -13,7 +13,7 @@ const aws_cloudwatch_actions_1 = require("aws-cdk-lib/aws-cloudwatch-actions");
|
|
|
13
13
|
* SNS topic.
|
|
14
14
|
*/
|
|
15
15
|
class MonitoredQueue extends constructs_1.Construct {
|
|
16
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.monitoredQueue.MonitoredQueue", version: "0.0.
|
|
16
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.monitoredQueue.MonitoredQueue", version: "0.0.35" };
|
|
17
17
|
queue;
|
|
18
18
|
deadLetterQueue;
|
|
19
19
|
alarm;
|
|
@@ -7,7 +7,7 @@ const aws_iam_1 = require("aws-cdk-lib/aws-iam");
|
|
|
7
7
|
* A construct which represents a named role that a Lambda function can assume.
|
|
8
8
|
*/
|
|
9
9
|
class NamedLambdaRole extends aws_iam_1.Role {
|
|
10
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.namedLambdaRole.NamedLambdaRole", version: "0.0.
|
|
10
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.namedLambdaRole.NamedLambdaRole", version: "0.0.35" };
|
|
11
11
|
constructor(scope, id, props) {
|
|
12
12
|
super(scope, id, {
|
|
13
13
|
assumedBy: new aws_iam_1.ServicePrincipal("lambda.amazonaws.com"),
|
package/package.json
CHANGED
|
@@ -15,7 +15,7 @@ const aws_cdk_lib_1 = require("aws-cdk-lib");
|
|
|
15
15
|
* to the provider framework.
|
|
16
16
|
*/
|
|
17
17
|
class ProviderFunction extends constructs_1.Construct {
|
|
18
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.provider.ProviderFunction", version: "0.0.
|
|
18
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.provider.ProviderFunction", version: "0.0.35" };
|
|
19
19
|
_function;
|
|
20
20
|
_response;
|
|
21
21
|
constructor(scope, id, props) {
|
|
@@ -23,7 +23,7 @@ exports.VPC_LOOKUP_PROPS = {
|
|
|
23
23
|
* Helper for looking up the shared OrcaBus VPC.
|
|
24
24
|
*/
|
|
25
25
|
class OrcaBusVpc {
|
|
26
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.sharedConfig.networking.OrcaBusVpc", version: "0.0.
|
|
26
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.sharedConfig.networking.OrcaBusVpc", version: "0.0.35" };
|
|
27
27
|
/**
|
|
28
28
|
* The shared VPC that is used by OrcaBus.
|
|
29
29
|
* @param scope
|
|
@@ -42,7 +42,7 @@ exports.SHARED_SECURITY_GROUP_NAME = "OrcaBusSharedComputeSecurityGroup";
|
|
|
42
42
|
* Helper for looking up the shared compute security group by name.
|
|
43
43
|
*/
|
|
44
44
|
class OrcaBusSharedComputeSecurityGroup {
|
|
45
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.sharedConfig.networking.OrcaBusSharedComputeSecurityGroup", version: "0.0.
|
|
45
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.sharedConfig.networking.OrcaBusSharedComputeSecurityGroup", version: "0.0.35" };
|
|
46
46
|
/**
|
|
47
47
|
* The shared security group that is used by compute resources to access the database.
|
|
48
48
|
* @param scope
|
package/shared-config/slack.js
CHANGED
|
@@ -11,7 +11,7 @@ exports.SLACK_ALERTS_SNS_TOPIC = "AwsChatBotTopic-alerts";
|
|
|
11
11
|
* A helper class to construct the arn for the slack SNS topic.
|
|
12
12
|
*/
|
|
13
13
|
class SlackAlerts {
|
|
14
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.sharedConfig.slack.SlackAlerts", version: "0.0.
|
|
14
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.sharedConfig.slack.SlackAlerts", version: "0.0.35" };
|
|
15
15
|
/**
|
|
16
16
|
* Format the ARN for the slack alerts SNS topic for the current stack.
|
|
17
17
|
* @param stack
|