@orcabus/platform-cdk-constructs 0.0.34 → 0.0.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.jsii +2 -2
- package/api-gateway/api-gateway.js +1 -1
- package/deployment-stack-pipeline/artifact-bucket.js +1 -1
- package/deployment-stack-pipeline/pipeline.js +1 -1
- package/dynamodb/index.js +2 -2
- package/ecs/index.js +1 -1
- package/lambda/index.js +1 -1
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/data_sharing/__init__.py +70 -0
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/data_sharing/globals.py +15 -0
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/data_sharing/models.py +28 -0
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/data_sharing/update_helpers.py +53 -0
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/__init__.py +12 -2
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/create_helpers.py +22 -5
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/globals.py +1 -1
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/job_helpers.py +16 -4
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/models.py +81 -20
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/query_helpers.py +25 -14
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/update_helpers.py +42 -18
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/workflow_helpers.py +3 -2
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq_unarchiving/__init__.py +4 -1
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq_unarchiving/create_helpers.py +2 -2
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq_unarchiving/globals.py +0 -8
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq_unarchiving/models.py +13 -14
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq_unarchiving/query_helpers.py +3 -3
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq_unarchiving/update_helpers.py +3 -3
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/filemanager/__init__.py +2 -0
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/filemanager/file_helpers.py +18 -11
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/filemanager/globals.py +0 -28
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/filemanager/models.py +19 -2
- package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/utils/requests_helpers.py +9 -2
- package/monitored-queue/index.js +1 -1
- package/named-lambda-role/index.js +1 -1
- package/package.json +1 -1
- package/provider-function/index.js +1 -1
- package/shared-config/networking.js +2 -2
- package/shared-config/slack.js +1 -1
package/.jsii
CHANGED
|
@@ -6756,6 +6756,6 @@
|
|
|
6756
6756
|
"symbolId": "shared-config/slack:SlackAlerts"
|
|
6757
6757
|
}
|
|
6758
6758
|
},
|
|
6759
|
-
"version": "0.0.
|
|
6760
|
-
"fingerprint": "
|
|
6759
|
+
"version": "0.0.35",
|
|
6760
|
+
"fingerprint": "dgr8YQrliJh0hWpEcsepSzBVhIznw5JYimaBU1p7VKg="
|
|
6761
6761
|
}
|
|
@@ -15,7 +15,7 @@ const aws_lambda_1 = require("aws-cdk-lib/aws-lambda");
|
|
|
15
15
|
const aws_route53_targets_1 = require("aws-cdk-lib/aws-route53-targets");
|
|
16
16
|
const config_1 = require("./config");
|
|
17
17
|
class OrcaBusApiGateway extends constructs_1.Construct {
|
|
18
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.apigateway.OrcaBusApiGateway", version: "0.0.
|
|
18
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.apigateway.OrcaBusApiGateway", version: "0.0.35" };
|
|
19
19
|
/**
|
|
20
20
|
* The AWS region where the API Gateway is deployed.
|
|
21
21
|
*/
|
|
@@ -13,7 +13,7 @@ exports.CROSS_DEPLOYMENT_ARTIFACT_BUCKET_NAME = "orcabus-cross-deployment-codepi
|
|
|
13
13
|
exports.CROSS_DEPLOYMENT_ARTIFACT_KMS_ALIAS = "orcabus-cross-deployment-codepipeline-artifact";
|
|
14
14
|
exports.CROSS_DEPLOYMENT_ARTIFACT_KMS_ARN_SSM_PARAMETER_NAME = "/orcabus/deployment-stack-pipeline/artifact-bucket/kms-key-arn";
|
|
15
15
|
class CrossDeploymentArtifactBucket extends constructs_1.Construct {
|
|
16
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.deploymentPipeline.CrossDeploymentArtifactBucket", version: "0.0.
|
|
16
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.deploymentPipeline.CrossDeploymentArtifactBucket", version: "0.0.35" };
|
|
17
17
|
/**
|
|
18
18
|
* The S3 bucket used to store artifacts for cross-deployment pipelines.
|
|
19
19
|
*/
|
|
@@ -32,7 +32,7 @@ exports.DEFAULT_SYNTH_STEP_PARTIAL_BUILD_SPEC = {
|
|
|
32
32
|
* before using this construct.
|
|
33
33
|
*/
|
|
34
34
|
class DeploymentStackPipeline extends constructs_1.Construct {
|
|
35
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.deploymentPipeline.DeploymentStackPipeline", version: "0.0.
|
|
35
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.deploymentPipeline.DeploymentStackPipeline", version: "0.0.35" };
|
|
36
36
|
/**
|
|
37
37
|
* The code pipeline construct that is created.
|
|
38
38
|
*/
|
package/dynamodb/index.js
CHANGED
|
@@ -40,7 +40,7 @@ const dynamodb = __importStar(require("aws-cdk-lib/aws-dynamodb"));
|
|
|
40
40
|
const aws_cdk_lib_1 = require("aws-cdk-lib");
|
|
41
41
|
const config_1 = require("./config");
|
|
42
42
|
class DynamoDbPartitionedConstruct extends constructs_1.Construct {
|
|
43
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.dynamodb.DynamoDbPartitionedConstruct", version: "0.0.
|
|
43
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.dynamodb.DynamoDbPartitionedConstruct", version: "0.0.35" };
|
|
44
44
|
table;
|
|
45
45
|
constructor(scope, id, props) {
|
|
46
46
|
super(scope, id);
|
|
@@ -75,7 +75,7 @@ class DynamoDbPartitionedConstruct extends constructs_1.Construct {
|
|
|
75
75
|
}
|
|
76
76
|
exports.DynamoDbPartitionedConstruct = DynamoDbPartitionedConstruct;
|
|
77
77
|
class DynamoDbNonPartitionedConstruct extends constructs_1.Construct {
|
|
78
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.dynamodb.DynamoDbNonPartitionedConstruct", version: "0.0.
|
|
78
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.dynamodb.DynamoDbNonPartitionedConstruct", version: "0.0.35" };
|
|
79
79
|
table;
|
|
80
80
|
constructor(scope, id, props) {
|
|
81
81
|
super(scope, id);
|
package/ecs/index.js
CHANGED
|
@@ -55,7 +55,7 @@ exports.LAMBDA_ARCHITECTURE_MAP = {
|
|
|
55
55
|
['ARM64']: lambda.Architecture.ARM_64
|
|
56
56
|
};
|
|
57
57
|
class EcsFargateTaskConstruct extends constructs_1.Construct {
|
|
58
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.ecs.EcsFargateTaskConstruct", version: "0.0.
|
|
58
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.ecs.EcsFargateTaskConstruct", version: "0.0.35" };
|
|
59
59
|
cluster;
|
|
60
60
|
taskDefinition;
|
|
61
61
|
taskExecutionRole;
|
package/lambda/index.js
CHANGED
|
@@ -56,7 +56,7 @@ function getPythonUvDockerImage() {
|
|
|
56
56
|
return aws_cdk_lib_1.DockerImage.fromBuild(path_1.default.join(__dirname, 'build_python'));
|
|
57
57
|
}
|
|
58
58
|
class PythonUvFunction extends aws_lambda_python_alpha_1.PythonFunction {
|
|
59
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.lambda.PythonUvFunction", version: "0.0.
|
|
59
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.lambda.PythonUvFunction", version: "0.0.35" };
|
|
60
60
|
// Class constructs, to be used for caching the layers
|
|
61
61
|
// This means that if there are multiple lambdas throughout the stack
|
|
62
62
|
// They will all use the same layer
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# FIMXE Implement the following imports
|
|
2
|
+
from typing import Optional, Dict
|
|
3
|
+
|
|
4
|
+
from .globals import DATA_SHARING_SUBDOMAIN_NAME
|
|
5
|
+
from ..utils.requests_helpers import get_url, get_request, get_request_response_results, patch_request
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
# Get url for the subdomain
|
|
9
|
+
def get_data_sharing_url(endpoint: str) -> str:
|
|
10
|
+
"""
|
|
11
|
+
Get the URL for the data sharing endpoint
|
|
12
|
+
:param endpoint:
|
|
13
|
+
:return:
|
|
14
|
+
"""
|
|
15
|
+
return get_url(
|
|
16
|
+
endpoint=endpoint,
|
|
17
|
+
subdomain=DATA_SHARING_SUBDOMAIN_NAME,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
# Wrappers
|
|
22
|
+
def get_data_sharing_request(
|
|
23
|
+
endpoint: str,
|
|
24
|
+
params: Optional[Dict] = None,
|
|
25
|
+
):
|
|
26
|
+
return get_request(
|
|
27
|
+
url=get_data_sharing_url(endpoint),
|
|
28
|
+
params=params
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def get_data_sharing_request_response_results(
|
|
33
|
+
endpoint: str,
|
|
34
|
+
params: Optional[Dict] = None,
|
|
35
|
+
):
|
|
36
|
+
return get_request_response_results(
|
|
37
|
+
url=get_data_sharing_url(endpoint),
|
|
38
|
+
params=params
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def data_sharing_patch_request(
|
|
43
|
+
endpoint: str,
|
|
44
|
+
params: Optional[Dict] = None,
|
|
45
|
+
):
|
|
46
|
+
return patch_request(
|
|
47
|
+
url=get_data_sharing_url(endpoint),
|
|
48
|
+
params=params
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def data_sharing_post_request(
|
|
53
|
+
endpoint: str,
|
|
54
|
+
params: Optional[Dict] = None,
|
|
55
|
+
):
|
|
56
|
+
return patch_request(
|
|
57
|
+
url=get_data_sharing_url(endpoint),
|
|
58
|
+
params=params
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
from .update_helpers import (
|
|
63
|
+
update_package_status,
|
|
64
|
+
update_push_job_status,
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
__all__ = [
|
|
68
|
+
"update_package_status",
|
|
69
|
+
"update_push_job_status",
|
|
70
|
+
]
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Imports
|
|
2
|
+
import re
|
|
3
|
+
from typing import Literal
|
|
4
|
+
|
|
5
|
+
# AWS PARAMETERS
|
|
6
|
+
DATA_SHARING_SUBDOMAIN_NAME = "data-sharing"
|
|
7
|
+
|
|
8
|
+
# API ENDPOINTS
|
|
9
|
+
PACKAGE_ENDPOINT = "api/v1/package"
|
|
10
|
+
PUSH_ENDPOINT = "api/v1/push"
|
|
11
|
+
|
|
12
|
+
# REGEX
|
|
13
|
+
ORCABUS_ULID_REGEX_MATCH = re.compile(r'^[a-z0-9]{3}\.[A-Z0-9]{26}$')
|
|
14
|
+
|
|
15
|
+
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from typing import Literal, TypedDict, NotRequired
|
|
2
|
+
|
|
3
|
+
PackageStatusType = Literal[
|
|
4
|
+
"PENDING",
|
|
5
|
+
"RUNNING",
|
|
6
|
+
"FAILED",
|
|
7
|
+
"ABORTED",
|
|
8
|
+
"SUCCEEDED",
|
|
9
|
+
]
|
|
10
|
+
|
|
11
|
+
PushJobStatusType = Literal[
|
|
12
|
+
"PENDING",
|
|
13
|
+
"RUNNING",
|
|
14
|
+
"FAILED",
|
|
15
|
+
"ABORTED",
|
|
16
|
+
"SUCCEEDED",
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class PackageObject(TypedDict):
|
|
22
|
+
id: str
|
|
23
|
+
packageName: str
|
|
24
|
+
stepsExecutionArn: str
|
|
25
|
+
status: PackageStatusType
|
|
26
|
+
requestTime: str
|
|
27
|
+
completionTime: NotRequired[str]
|
|
28
|
+
hasExpired: NotRequired[bool]
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
from typing import Optional
|
|
2
|
+
|
|
3
|
+
from . import data_sharing_patch_request
|
|
4
|
+
from .globals import PACKAGE_ENDPOINT, PUSH_ENDPOINT
|
|
5
|
+
from .models import PackageStatusType, PackageObject, PushJobStatusType
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def update_package_status(
|
|
9
|
+
package_id: str,
|
|
10
|
+
package_status: PackageStatusType,
|
|
11
|
+
error_message: Optional[str] = None
|
|
12
|
+
) -> PackageObject:
|
|
13
|
+
"""
|
|
14
|
+
Add QC stats to a fastq_id.
|
|
15
|
+
|
|
16
|
+
:param package_id: The package id
|
|
17
|
+
:param package_status: The package status to set
|
|
18
|
+
:param error_message: Optional error message
|
|
19
|
+
"""
|
|
20
|
+
return data_sharing_patch_request(
|
|
21
|
+
f"{PACKAGE_ENDPOINT}/{package_id}",
|
|
22
|
+
params=dict(filter(
|
|
23
|
+
lambda x: x[1] is not None,
|
|
24
|
+
{
|
|
25
|
+
"status": package_status,
|
|
26
|
+
"error_message": error_message
|
|
27
|
+
}.items()
|
|
28
|
+
))
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def update_push_job_status(
|
|
33
|
+
push_job_id: str,
|
|
34
|
+
push_job_status: PushJobStatusType,
|
|
35
|
+
error_message: Optional[str] = None
|
|
36
|
+
) -> PackageObject:
|
|
37
|
+
"""
|
|
38
|
+
Add push job status to a push job.
|
|
39
|
+
|
|
40
|
+
:param push_job_id: The push job id
|
|
41
|
+
:param push_job_status: The push job status to set
|
|
42
|
+
:param error_message: Optional error message
|
|
43
|
+
"""
|
|
44
|
+
return data_sharing_patch_request(
|
|
45
|
+
f"{PUSH_ENDPOINT}/{push_job_id}",
|
|
46
|
+
params=dict(filter(
|
|
47
|
+
lambda x: x[1] is not None,
|
|
48
|
+
{
|
|
49
|
+
"status": push_job_status,
|
|
50
|
+
"error_message": error_message
|
|
51
|
+
}.items()
|
|
52
|
+
))
|
|
53
|
+
)
|
|
@@ -70,14 +70,16 @@ def fastq_post_request(
|
|
|
70
70
|
# Create
|
|
71
71
|
from .create_helpers import (
|
|
72
72
|
create_fastq_set_object,
|
|
73
|
-
create_fastq_list_row_object
|
|
73
|
+
create_fastq_list_row_object,
|
|
74
|
+
create_fastq_object,
|
|
74
75
|
)
|
|
75
76
|
|
|
76
77
|
# Job
|
|
77
78
|
from .job_helpers import (
|
|
78
79
|
run_qc_stats,
|
|
79
80
|
run_ntsm,
|
|
80
|
-
run_file_compression_stats
|
|
81
|
+
run_file_compression_stats,
|
|
82
|
+
run_read_count_stats
|
|
81
83
|
)
|
|
82
84
|
|
|
83
85
|
# Query
|
|
@@ -97,6 +99,7 @@ from .query_helpers import (
|
|
|
97
99
|
get_fastqs_in_individual,
|
|
98
100
|
get_fastqs_in_project,
|
|
99
101
|
get_fastq_list_rows_in_fastq_set,
|
|
102
|
+
get_fastqs_in_fastq_set,
|
|
100
103
|
get_fastq_jobs,
|
|
101
104
|
)
|
|
102
105
|
|
|
@@ -111,7 +114,9 @@ from .update_helpers import (
|
|
|
111
114
|
validate_fastq,
|
|
112
115
|
invalidate_fastq,
|
|
113
116
|
link_fastq_list_row_to_fastq_set,
|
|
117
|
+
link_fastq_to_fastq_set,
|
|
114
118
|
unlink_fastq_list_row_from_fastq_set,
|
|
119
|
+
unlink_fastq_from_fastq_set,
|
|
115
120
|
allow_additional_fastqs_to_fastq_set,
|
|
116
121
|
disallow_additional_fastqs_to_fastq_set,
|
|
117
122
|
set_is_current_fastq_set,
|
|
@@ -135,10 +140,12 @@ __all__ = [
|
|
|
135
140
|
# Create
|
|
136
141
|
"create_fastq_set_object",
|
|
137
142
|
"create_fastq_list_row_object",
|
|
143
|
+
"create_fastq_object",
|
|
138
144
|
# Job
|
|
139
145
|
"run_qc_stats",
|
|
140
146
|
"run_ntsm",
|
|
141
147
|
"run_file_compression_stats",
|
|
148
|
+
"run_read_count_stats",
|
|
142
149
|
# Query
|
|
143
150
|
"get_fastq",
|
|
144
151
|
"get_fastq_set",
|
|
@@ -155,6 +162,7 @@ __all__ = [
|
|
|
155
162
|
"get_fastqs_in_individual",
|
|
156
163
|
"get_fastqs_in_project",
|
|
157
164
|
"get_fastq_list_rows_in_fastq_set",
|
|
165
|
+
"get_fastqs_in_fastq_set",
|
|
158
166
|
"get_fastq_jobs",
|
|
159
167
|
# Updaters
|
|
160
168
|
"add_qc_stats",
|
|
@@ -166,7 +174,9 @@ __all__ = [
|
|
|
166
174
|
"validate_fastq",
|
|
167
175
|
"invalidate_fastq",
|
|
168
176
|
"link_fastq_list_row_to_fastq_set",
|
|
177
|
+
"link_fastq_to_fastq_set",
|
|
169
178
|
"unlink_fastq_list_row_from_fastq_set",
|
|
179
|
+
"unlink_fastq_from_fastq_set",
|
|
170
180
|
"allow_additional_fastqs_to_fastq_set",
|
|
171
181
|
"disallow_additional_fastqs_to_fastq_set",
|
|
172
182
|
"set_is_current_fastq_set",
|
|
@@ -17,18 +17,30 @@ from typing import Unpack
|
|
|
17
17
|
|
|
18
18
|
# Local imports
|
|
19
19
|
from . import fastq_post_request
|
|
20
|
-
from .globals import
|
|
21
|
-
from .models import
|
|
20
|
+
from .globals import FASTQ_ENDPOINT, FASTQ_SET_ENDPOINT
|
|
21
|
+
from .models import Fastq, FastqSet, FastqCreate, FastqSetCreate
|
|
22
22
|
|
|
23
23
|
|
|
24
|
-
def create_fastq_list_row_object(**kwargs: Unpack[
|
|
24
|
+
def create_fastq_list_row_object(**kwargs: Unpack[FastqCreate]) -> Fastq:
|
|
25
|
+
"""
|
|
26
|
+
DEPRECATED: Use create_fastq_object instead.
|
|
27
|
+
"""
|
|
28
|
+
return create_fastq_object(**kwargs)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def create_fastq_object(**kwargs: Unpack[FastqCreate]) -> Fastq:
|
|
25
32
|
"""
|
|
26
33
|
Add a fastq list row object to the database.
|
|
27
34
|
Returns the created fastq list row object
|
|
28
35
|
"""
|
|
29
|
-
|
|
36
|
+
# Raise error if any of the kwargs are not in the FastqCreate
|
|
37
|
+
for key in kwargs.keys():
|
|
38
|
+
if key not in FastqCreate.__annotations__:
|
|
39
|
+
raise ValueError(f"Invalid parameter: {key}")
|
|
40
|
+
|
|
41
|
+
return Fastq(
|
|
30
42
|
**fastq_post_request(
|
|
31
|
-
endpoint=
|
|
43
|
+
endpoint=FASTQ_ENDPOINT,
|
|
32
44
|
params=dict(kwargs)
|
|
33
45
|
)
|
|
34
46
|
)
|
|
@@ -39,6 +51,11 @@ def create_fastq_set_object(**kwargs: Unpack[FastqSetCreate]) -> FastqSet:
|
|
|
39
51
|
Add a fastq set object to the database.
|
|
40
52
|
Returns the created fastq set object
|
|
41
53
|
"""
|
|
54
|
+
# Raise error if any of the kwargs are not in the FastqCreate
|
|
55
|
+
for key in kwargs.keys():
|
|
56
|
+
if key not in FastqSetCreate.__annotations__:
|
|
57
|
+
raise ValueError(f"Invalid parameter: {key}")
|
|
58
|
+
|
|
42
59
|
return FastqSet(
|
|
43
60
|
**fastq_post_request(
|
|
44
61
|
endpoint=FASTQ_SET_ENDPOINT,
|
|
@@ -11,7 +11,7 @@ Update helpers for the update script.
|
|
|
11
11
|
|
|
12
12
|
# Local imports
|
|
13
13
|
from . import fastq_patch_request
|
|
14
|
-
from .globals import
|
|
14
|
+
from .globals import FASTQ_ENDPOINT
|
|
15
15
|
from .models import Job
|
|
16
16
|
|
|
17
17
|
|
|
@@ -23,7 +23,7 @@ def run_qc_stats(fastq_id: str) -> Job:
|
|
|
23
23
|
"""
|
|
24
24
|
return Job(
|
|
25
25
|
**fastq_patch_request(
|
|
26
|
-
f"{
|
|
26
|
+
f"{FASTQ_ENDPOINT}/{fastq_id}:runQcStats"
|
|
27
27
|
)
|
|
28
28
|
)
|
|
29
29
|
|
|
@@ -36,7 +36,7 @@ def run_ntsm(fastq_id: str) -> Job:
|
|
|
36
36
|
"""
|
|
37
37
|
return Job(
|
|
38
38
|
**fastq_patch_request(
|
|
39
|
-
f"{
|
|
39
|
+
f"{FASTQ_ENDPOINT}/{fastq_id}:runNtsm"
|
|
40
40
|
)
|
|
41
41
|
)
|
|
42
42
|
|
|
@@ -48,6 +48,18 @@ def run_file_compression_stats(fastq_id: str) -> Job:
|
|
|
48
48
|
"""
|
|
49
49
|
return Job(
|
|
50
50
|
**fastq_patch_request(
|
|
51
|
-
f"{
|
|
51
|
+
f"{FASTQ_ENDPOINT}/{fastq_id}:runFileCompressionInformation"
|
|
52
|
+
)
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
def run_read_count_stats(fastq_id: str) -> Job:
|
|
56
|
+
"""
|
|
57
|
+
Run file compression stats for a fastq_id.
|
|
58
|
+
|
|
59
|
+
:param fastq_id: Fastq str
|
|
60
|
+
"""
|
|
61
|
+
return Job(
|
|
62
|
+
**fastq_patch_request(
|
|
63
|
+
f"{FASTQ_ENDPOINT}/{fastq_id}:runReadCountInformation"
|
|
52
64
|
)
|
|
53
65
|
)
|
|
@@ -40,22 +40,27 @@ from typing import (
|
|
|
40
40
|
TypedDict,
|
|
41
41
|
Optional,
|
|
42
42
|
Dict,
|
|
43
|
-
List, NotRequired, Union,
|
|
43
|
+
List, NotRequired, Union,
|
|
44
|
+
Literal
|
|
44
45
|
)
|
|
45
46
|
from datetime import datetime
|
|
46
47
|
|
|
47
48
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
49
|
+
JobType = Literal[
|
|
50
|
+
'QC',
|
|
51
|
+
'FILE_COMPRESSION',
|
|
52
|
+
'NTSM',
|
|
53
|
+
'READ_COUNT',
|
|
54
|
+
]
|
|
55
|
+
|
|
52
56
|
|
|
57
|
+
JobStatus = Literal[
|
|
58
|
+
"PENDING",
|
|
59
|
+
"RUNNING",
|
|
60
|
+
"FAILED",
|
|
61
|
+
"SUCCEEDED",
|
|
62
|
+
]
|
|
53
63
|
|
|
54
|
-
class JobStatus(Enum):
|
|
55
|
-
PENDING = "PENDING"
|
|
56
|
-
RUNNING = "RUNNING"
|
|
57
|
-
FAILED = "FAILED"
|
|
58
|
-
SUCCEEDED = "SUCCEEDED"
|
|
59
64
|
|
|
60
65
|
|
|
61
66
|
class FileStorageObject(TypedDict):
|
|
@@ -81,6 +86,7 @@ class Library(TypedDict):
|
|
|
81
86
|
libraryId: str
|
|
82
87
|
|
|
83
88
|
|
|
89
|
+
# Deprecated: Use FastqCreate instead
|
|
84
90
|
class FastqListRowCreate(TypedDict):
|
|
85
91
|
fastqSetId: Optional[str]
|
|
86
92
|
index: str
|
|
@@ -98,6 +104,7 @@ class FastqListRowCreate(TypedDict):
|
|
|
98
104
|
ntsm: Optional[FileStorageObject]
|
|
99
105
|
|
|
100
106
|
|
|
107
|
+
# Deprecated: Use Fastq instead
|
|
101
108
|
class FastqListRow(TypedDict):
|
|
102
109
|
id: str
|
|
103
110
|
fastqSetId: Optional[str]
|
|
@@ -116,9 +123,44 @@ class FastqListRow(TypedDict):
|
|
|
116
123
|
ntsm: Optional[FileStorageObject]
|
|
117
124
|
|
|
118
125
|
|
|
126
|
+
class Fastq(TypedDict):
|
|
127
|
+
id: str
|
|
128
|
+
fastqSetId: Optional[str]
|
|
129
|
+
index: str
|
|
130
|
+
lane: int
|
|
131
|
+
instrumentRunId: str
|
|
132
|
+
library: Library
|
|
133
|
+
platform: Optional[str]
|
|
134
|
+
center: Optional[str]
|
|
135
|
+
date: Optional[datetime]
|
|
136
|
+
readSet: Optional[ReadSet]
|
|
137
|
+
qc: Optional[Dict]
|
|
138
|
+
readCount: Optional[int]
|
|
139
|
+
baseCountEst: Optional[int]
|
|
140
|
+
isValid: Optional[bool]
|
|
141
|
+
ntsm: Optional[FileStorageObject]
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
class FastqCreate(TypedDict):
|
|
145
|
+
fastqSetId: Optional[str]
|
|
146
|
+
index: str
|
|
147
|
+
lane: int
|
|
148
|
+
instrumentRunId: str
|
|
149
|
+
library: Library
|
|
150
|
+
platform: Optional[str]
|
|
151
|
+
center: Optional[str]
|
|
152
|
+
date: Optional[datetime]
|
|
153
|
+
readSet: Optional[ReadSet]
|
|
154
|
+
qc: Optional[Dict]
|
|
155
|
+
readCount: Optional[int]
|
|
156
|
+
baseCountEst: Optional[int]
|
|
157
|
+
isValid: Optional[bool]
|
|
158
|
+
ntsm: Optional[FileStorageObject]
|
|
159
|
+
|
|
160
|
+
|
|
119
161
|
class FastqSetCreate(TypedDict):
|
|
120
162
|
library: Library
|
|
121
|
-
fastqSet: List[Union[str,
|
|
163
|
+
fastqSet: List[Union[str, Fastq]]
|
|
122
164
|
allowAdditionalFastq: bool
|
|
123
165
|
isCurrentFastqSet: bool
|
|
124
166
|
|
|
@@ -126,7 +168,7 @@ class FastqSetCreate(TypedDict):
|
|
|
126
168
|
class FastqSet(TypedDict):
|
|
127
169
|
id: str
|
|
128
170
|
library: Library
|
|
129
|
-
fastqSet: List[
|
|
171
|
+
fastqSet: List[Fastq]
|
|
130
172
|
allowAdditionalFastq: bool
|
|
131
173
|
isCurrentFastqSet: bool
|
|
132
174
|
|
|
@@ -176,15 +218,21 @@ class Job(TypedDict):
|
|
|
176
218
|
startTime: datetime
|
|
177
219
|
endTime: Optional[datetime]
|
|
178
220
|
|
|
179
|
-
|
|
221
|
+
# Deprecated: Use BoolLiteral instead
|
|
180
222
|
class BoolAllEnum(Enum):
|
|
181
223
|
ALL = "ALL"
|
|
182
224
|
true = True
|
|
183
225
|
false = False
|
|
184
226
|
|
|
227
|
+
BoolLiteral = Literal[
|
|
228
|
+
'ALL',
|
|
229
|
+
True,
|
|
230
|
+
False
|
|
231
|
+
]
|
|
232
|
+
|
|
185
233
|
|
|
186
234
|
class FastqGetResponseParameters(TypedDict):
|
|
187
|
-
includeS3Details: NotRequired[
|
|
235
|
+
includeS3Details: NotRequired[BoolLiteral]
|
|
188
236
|
|
|
189
237
|
|
|
190
238
|
class StandardQueryParameters(TypedDict):
|
|
@@ -224,15 +272,28 @@ InstrumentRunIdQueryParametersList = TypedDict(
|
|
|
224
272
|
)
|
|
225
273
|
|
|
226
274
|
|
|
227
|
-
|
|
275
|
+
# Deprecated: Use FastqQueryParameters instead
|
|
276
|
+
class FastqParameters(
|
|
277
|
+
StandardQueryParameters,
|
|
278
|
+
MetadataQueryParameter,
|
|
279
|
+
InstrumentRunIdQueryParameters,
|
|
280
|
+
MetadataQueryParametersList,
|
|
281
|
+
InstrumentRunIdQueryParametersList
|
|
282
|
+
):
|
|
283
|
+
valid: NotRequired[BoolLiteral]
|
|
284
|
+
includeS3Details: NotRequired[BoolLiteral]
|
|
285
|
+
fastqSetId: NotRequired[str]
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
class FastqQueryParameters(
|
|
228
289
|
StandardQueryParameters,
|
|
229
290
|
MetadataQueryParameter,
|
|
230
291
|
InstrumentRunIdQueryParameters,
|
|
231
292
|
MetadataQueryParametersList,
|
|
232
293
|
InstrumentRunIdQueryParametersList
|
|
233
294
|
):
|
|
234
|
-
valid: NotRequired[
|
|
235
|
-
includeS3Details: NotRequired[
|
|
295
|
+
valid: NotRequired[BoolLiteral]
|
|
296
|
+
includeS3Details: NotRequired[BoolLiteral]
|
|
236
297
|
fastqSetId: NotRequired[str]
|
|
237
298
|
|
|
238
299
|
|
|
@@ -242,9 +303,9 @@ class FastqSetQueryParameters(
|
|
|
242
303
|
MetadataQueryParametersList,
|
|
243
304
|
InstrumentRunIdQueryParameters
|
|
244
305
|
):
|
|
245
|
-
currentFastqSet: NotRequired[
|
|
246
|
-
allowAdditionalFastq: NotRequired[
|
|
247
|
-
includeS3Details: NotRequired[
|
|
306
|
+
currentFastqSet: NotRequired[BoolLiteral]
|
|
307
|
+
allowAdditionalFastq: NotRequired[BoolLiteral]
|
|
308
|
+
includeS3Details: NotRequired[BoolLiteral]
|
|
248
309
|
|
|
249
310
|
# Additional types
|
|
250
311
|
VALID_BATCH_KEYS = Literal[
|
|
@@ -25,20 +25,22 @@ from typing import List, Unpack
|
|
|
25
25
|
from fastapi.encoders import jsonable_encoder
|
|
26
26
|
|
|
27
27
|
from . import get_fastq_request_response_results, get_fastq_request
|
|
28
|
-
from .globals import
|
|
29
|
-
from .models import
|
|
30
|
-
|
|
28
|
+
from .globals import FASTQ_ENDPOINT, FASTQ_SET_ENDPOINT
|
|
29
|
+
from .models import (
|
|
30
|
+
FastqSet, Job, FastqParameters, FastqSetQueryParameters,
|
|
31
|
+
FastqGetResponseParameters, VALID_BATCH_KEYS, Fastq
|
|
32
|
+
)
|
|
31
33
|
|
|
32
34
|
|
|
33
|
-
def get_fastq(fastq_id: str, **kwargs: Unpack[FastqGetResponseParameters]) ->
|
|
35
|
+
def get_fastq(fastq_id: str, **kwargs: Unpack[FastqGetResponseParameters]) -> Fastq:
|
|
34
36
|
# Raise error if any of the kwargs are not in the FastqSetQueryParameters
|
|
35
37
|
for key in kwargs.keys():
|
|
36
38
|
if key not in FastqGetResponseParameters.__annotations__:
|
|
37
39
|
raise ValueError(f"Invalid parameter: {key}")
|
|
38
40
|
|
|
39
|
-
return
|
|
41
|
+
return Fastq(
|
|
40
42
|
**get_fastq_request(
|
|
41
|
-
f"{
|
|
43
|
+
f"{FASTQ_ENDPOINT}/{fastq_id}",
|
|
42
44
|
params=dict(kwargs)
|
|
43
45
|
)
|
|
44
46
|
)
|
|
@@ -67,17 +69,17 @@ def get_fastq_set(
|
|
|
67
69
|
)
|
|
68
70
|
|
|
69
71
|
|
|
70
|
-
def get_fastqs(**kwargs: Unpack[
|
|
72
|
+
def get_fastqs(**kwargs: Unpack[FastqParameters]) -> List[Fastq]:
|
|
71
73
|
"""
|
|
72
74
|
Get all fastqs
|
|
73
75
|
"""
|
|
74
|
-
# Raise error if any of the kwargs are not in the
|
|
76
|
+
# Raise error if any of the kwargs are not in the FastqParameters
|
|
75
77
|
for key in kwargs.keys():
|
|
76
|
-
if key not in
|
|
78
|
+
if key not in FastqParameters.__annotations__:
|
|
77
79
|
raise ValueError(f"Invalid parameter: {key}")
|
|
78
80
|
|
|
79
81
|
return get_fastq_request_response_results(
|
|
80
|
-
|
|
82
|
+
FASTQ_ENDPOINT,
|
|
81
83
|
params=dict(kwargs)
|
|
82
84
|
)
|
|
83
85
|
|
|
@@ -89,7 +91,7 @@ def get_fastq_sets(**kwargs: Unpack[FastqSetQueryParameters]) -> List[FastqSet]:
|
|
|
89
91
|
:param kwargs:
|
|
90
92
|
:return:
|
|
91
93
|
"""
|
|
92
|
-
# Raise error if any of the kwargs are not in the
|
|
94
|
+
# Raise error if any of the kwargs are not in the FastqParameters
|
|
93
95
|
for key in kwargs.keys():
|
|
94
96
|
if key not in FastqSetQueryParameters.__annotations__:
|
|
95
97
|
raise ValueError(f"Invalid parameter: {key}")
|
|
@@ -123,7 +125,7 @@ def get_fastqs_batched(
|
|
|
123
125
|
item_list: List[str],
|
|
124
126
|
batch_size: int = 100,
|
|
125
127
|
**kwargs
|
|
126
|
-
) -> List[
|
|
128
|
+
) -> List[Fastq]:
|
|
127
129
|
"""
|
|
128
130
|
Get all fastqs in a list of libraries
|
|
129
131
|
"""
|
|
@@ -153,7 +155,7 @@ def get_fastqs_batched(
|
|
|
153
155
|
|
|
154
156
|
def get_fastqs_in_library_list(
|
|
155
157
|
library_id_list: List[str]
|
|
156
|
-
) -> List[
|
|
158
|
+
) -> List[Fastq]:
|
|
157
159
|
"""
|
|
158
160
|
Get all fastqs in a list of libraries
|
|
159
161
|
"""
|
|
@@ -229,8 +231,17 @@ def get_fastqs_in_project(project_id):
|
|
|
229
231
|
|
|
230
232
|
|
|
231
233
|
def get_fastq_list_rows_in_fastq_set(fastq_set_id):
|
|
234
|
+
"""
|
|
235
|
+
DEPRECATED: Use get_fastqs_in_fastq_set instead
|
|
236
|
+
"""
|
|
237
|
+
return get_fastqs_in_fastq_set(fastq_set_id)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def get_fastqs_in_fastq_set(fastq_set_id: str) -> List[Fastq]:
|
|
232
241
|
"""
|
|
233
242
|
Get all fastqs in a fastq set
|
|
243
|
+
:param fastq_set_id:
|
|
244
|
+
:return:
|
|
234
245
|
"""
|
|
235
246
|
return get_fastqs(
|
|
236
247
|
fastqSetId=fastq_set_id
|
|
@@ -244,6 +255,6 @@ def get_fastq_jobs(fastq_id: str) -> List[Job]:
|
|
|
244
255
|
return list(map(
|
|
245
256
|
lambda job_iter_: Job(**job_iter_),
|
|
246
257
|
get_fastq_request_response_results(
|
|
247
|
-
f"{
|
|
258
|
+
f"{FASTQ_ENDPOINT}/{fastq_id}/jobs"
|
|
248
259
|
)
|
|
249
260
|
))
|
|
@@ -15,15 +15,15 @@ Update helpers for the update script.
|
|
|
15
15
|
|
|
16
16
|
# Local imports
|
|
17
17
|
from . import fastq_patch_request
|
|
18
|
-
from .globals import
|
|
18
|
+
from .globals import FASTQ_ENDPOINT, FASTQ_SET_ENDPOINT
|
|
19
19
|
from .models import (
|
|
20
|
-
QcStats,
|
|
20
|
+
QcStats, Fastq, ReadCount,
|
|
21
21
|
FileCompressionInformation, FileStorageObject, ReadSet,
|
|
22
22
|
FastqSet
|
|
23
23
|
)
|
|
24
24
|
|
|
25
25
|
|
|
26
|
-
def add_qc_stats(fastq_id: str, qc_stats: QcStats) ->
|
|
26
|
+
def add_qc_stats(fastq_id: str, qc_stats: QcStats) -> Fastq:
|
|
27
27
|
"""
|
|
28
28
|
Add QC stats to a fastq_id.
|
|
29
29
|
|
|
@@ -36,12 +36,12 @@ def add_qc_stats(fastq_id: str, qc_stats: QcStats) -> FastqListRow:
|
|
|
36
36
|
raise ValueError(f"Invalid parameter: {key}")
|
|
37
37
|
|
|
38
38
|
return fastq_patch_request(
|
|
39
|
-
f"{
|
|
39
|
+
f"{FASTQ_ENDPOINT}/{fastq_id}/addQcStats",
|
|
40
40
|
params=dict(qc_stats)
|
|
41
41
|
)
|
|
42
42
|
|
|
43
43
|
|
|
44
|
-
def add_read_count(fastq_id: str, read_count: ReadCount) ->
|
|
44
|
+
def add_read_count(fastq_id: str, read_count: ReadCount) -> Fastq:
|
|
45
45
|
"""
|
|
46
46
|
Add read count to a fastq id
|
|
47
47
|
:param fastq_id:
|
|
@@ -53,12 +53,12 @@ def add_read_count(fastq_id: str, read_count: ReadCount) -> FastqListRow:
|
|
|
53
53
|
raise ValueError(f"Invalid parameter: {key}")
|
|
54
54
|
|
|
55
55
|
return fastq_patch_request(
|
|
56
|
-
f"{
|
|
56
|
+
f"{FASTQ_ENDPOINT}/{fastq_id}/addReadCount",
|
|
57
57
|
params=dict(read_count)
|
|
58
58
|
)
|
|
59
59
|
|
|
60
60
|
|
|
61
|
-
def add_file_compression_information(fastq_id: str, file_compression_information: FileCompressionInformation) ->
|
|
61
|
+
def add_file_compression_information(fastq_id: str, file_compression_information: FileCompressionInformation) -> Fastq:
|
|
62
62
|
"""
|
|
63
63
|
Add file compression information to a fastq id
|
|
64
64
|
:param fastq_id:
|
|
@@ -70,12 +70,12 @@ def add_file_compression_information(fastq_id: str, file_compression_information
|
|
|
70
70
|
raise ValueError(f"Invalid parameter: {key}")
|
|
71
71
|
|
|
72
72
|
return fastq_patch_request(
|
|
73
|
-
f"{
|
|
73
|
+
f"{FASTQ_ENDPOINT}/{fastq_id}/addFileCompressionInformation",
|
|
74
74
|
params=dict(file_compression_information)
|
|
75
75
|
)
|
|
76
76
|
|
|
77
77
|
|
|
78
|
-
def add_ntsm_storage_object(fastq_id: str, ntsm_fastq_storage_object: FileStorageObject) ->
|
|
78
|
+
def add_ntsm_storage_object(fastq_id: str, ntsm_fastq_storage_object: FileStorageObject) -> Fastq:
|
|
79
79
|
"""
|
|
80
80
|
Add a Ntsm storage object to a fastq id.
|
|
81
81
|
|
|
@@ -87,12 +87,12 @@ def add_ntsm_storage_object(fastq_id: str, ntsm_fastq_storage_object: FileStorag
|
|
|
87
87
|
raise ValueError(f"Invalid parameter: {key}")
|
|
88
88
|
|
|
89
89
|
return fastq_patch_request(
|
|
90
|
-
f"{
|
|
90
|
+
f"{FASTQ_ENDPOINT}/{fastq_id}/addNtsmStorageObject",
|
|
91
91
|
params=dict(ntsm_fastq_storage_object)
|
|
92
92
|
)
|
|
93
93
|
|
|
94
94
|
|
|
95
|
-
def add_read_set(fastq_id: str, read_set: ReadSet) ->
|
|
95
|
+
def add_read_set(fastq_id: str, read_set: ReadSet) -> Fastq:
|
|
96
96
|
"""
|
|
97
97
|
Add a read set to a fastq id.
|
|
98
98
|
|
|
@@ -104,48 +104,62 @@ def add_read_set(fastq_id: str, read_set: ReadSet) -> FastqListRow:
|
|
|
104
104
|
raise ValueError(f"Invalid parameter: {key}")
|
|
105
105
|
|
|
106
106
|
return fastq_patch_request(
|
|
107
|
-
f"{
|
|
107
|
+
f"{FASTQ_ENDPOINT}/{fastq_id}/addFastqPairStorageObject",
|
|
108
108
|
params=dict(read_set)
|
|
109
109
|
)
|
|
110
110
|
|
|
111
111
|
|
|
112
|
-
def detach_read_set(fastq_id: str) ->
|
|
112
|
+
def detach_read_set(fastq_id: str) -> Fastq:
|
|
113
113
|
"""
|
|
114
114
|
Detach a read set to a fastq id.
|
|
115
115
|
|
|
116
116
|
:param fastq_id: Fastq str
|
|
117
117
|
"""
|
|
118
118
|
return fastq_patch_request(
|
|
119
|
-
f"{
|
|
119
|
+
f"{FASTQ_ENDPOINT}/{fastq_id}/detachFastqPairStorageObject"
|
|
120
120
|
)
|
|
121
121
|
|
|
122
122
|
|
|
123
|
-
def validate_fastq(fastq_id: str) ->
|
|
123
|
+
def validate_fastq(fastq_id: str) -> Fastq:
|
|
124
124
|
"""
|
|
125
125
|
Validate a fastq id.
|
|
126
126
|
|
|
127
127
|
:param fastq_id: Fastq str
|
|
128
128
|
"""
|
|
129
129
|
return fastq_patch_request(
|
|
130
|
-
f"{
|
|
130
|
+
f"{FASTQ_ENDPOINT}/{fastq_id}/validate"
|
|
131
131
|
)
|
|
132
132
|
|
|
133
133
|
|
|
134
|
-
def invalidate_fastq(fastq_id: str) ->
|
|
134
|
+
def invalidate_fastq(fastq_id: str) -> Fastq:
|
|
135
135
|
"""
|
|
136
136
|
Invalidate a fastq id.
|
|
137
137
|
|
|
138
138
|
:param fastq_id: Fastq str
|
|
139
139
|
"""
|
|
140
140
|
return fastq_patch_request(
|
|
141
|
-
f"{
|
|
141
|
+
f"{FASTQ_ENDPOINT}/{fastq_id}/invalidate"
|
|
142
142
|
)
|
|
143
143
|
|
|
144
144
|
|
|
145
145
|
def link_fastq_list_row_to_fastq_set(fastq_id: str, fastq_set_id: str) -> FastqSet:
|
|
146
146
|
"""
|
|
147
|
+
Deprecated: Use `link_fastq_to_fast_set` instead.
|
|
147
148
|
Link a fastq id to a fastq set.
|
|
148
149
|
|
|
150
|
+
:param fastq_id:
|
|
151
|
+
:param fastq_set_id:
|
|
152
|
+
:return:
|
|
153
|
+
"""
|
|
154
|
+
return link_fastq_to_fastq_set(
|
|
155
|
+
fastq_id=fastq_id,
|
|
156
|
+
fastq_set_id=fastq_set_id
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def link_fastq_to_fastq_set(fastq_id: str, fastq_set_id: str) -> FastqSet:
|
|
161
|
+
"""
|
|
162
|
+
Link a fastq id to a fastq set.
|
|
149
163
|
:param fastq_id:
|
|
150
164
|
:param fastq_set_id:
|
|
151
165
|
:return:
|
|
@@ -156,6 +170,16 @@ def link_fastq_list_row_to_fastq_set(fastq_id: str, fastq_set_id: str) -> FastqS
|
|
|
156
170
|
|
|
157
171
|
|
|
158
172
|
def unlink_fastq_list_row_from_fastq_set(fastq_id: str, fastq_set_id: str) -> FastqSet:
|
|
173
|
+
"""
|
|
174
|
+
Deprecated: Use `unlink_fastq_from_fastq_set` instead.
|
|
175
|
+
"""
|
|
176
|
+
return unlink_fastq_from_fastq_set(
|
|
177
|
+
fastq_id=fastq_id,
|
|
178
|
+
fastq_set_id=fastq_set_id
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def unlink_fastq_from_fastq_set(fastq_id: str, fastq_set_id: str) -> FastqSet:
|
|
159
183
|
"""
|
|
160
184
|
Unlink a fastq id from a fastq set.
|
|
161
185
|
|
|
@@ -8,15 +8,16 @@ Workflow helpers - a collection of helper functions for the workflow
|
|
|
8
8
|
|
|
9
9
|
# Standard library imports
|
|
10
10
|
from typing import List
|
|
11
|
+
|
|
11
12
|
# Local imports
|
|
12
13
|
from . import get_fastq_request
|
|
13
|
-
from .globals import
|
|
14
|
+
from .globals import FASTQ_ENDPOINT, FASTQ_SET_ENDPOINT
|
|
14
15
|
from .models import FastqListRowDict
|
|
15
16
|
|
|
16
17
|
|
|
17
18
|
def to_fastq_list_row(fastq_id) -> FastqListRowDict:
|
|
18
19
|
return get_fastq_request(
|
|
19
|
-
f"{
|
|
20
|
+
f"{FASTQ_ENDPOINT}/{fastq_id}/toFastqListRow"
|
|
20
21
|
)
|
|
21
22
|
|
|
22
23
|
def to_fastq_list_rows(fastq_set_id: str) -> List[FastqListRowDict]:
|
|
@@ -6,7 +6,10 @@ from typing import Dict, Optional
|
|
|
6
6
|
# Local imports
|
|
7
7
|
from .globals import FASTQ_UNARCHIVING_SUBDOMAIN_NAME
|
|
8
8
|
from ..utils.requests_helpers import (
|
|
9
|
-
get_request_response_results,
|
|
9
|
+
get_request_response_results,
|
|
10
|
+
get_url,
|
|
11
|
+
patch_request,
|
|
12
|
+
get_request
|
|
10
13
|
)
|
|
11
14
|
|
|
12
15
|
|
package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq_unarchiving/create_helpers.py
CHANGED
|
@@ -16,12 +16,12 @@ def create_job(fastq_ids: List[str], job_type: Optional[JobType] = None) -> Job:
|
|
|
16
16
|
Create the job
|
|
17
17
|
"""
|
|
18
18
|
if job_type is None:
|
|
19
|
-
job_type =
|
|
19
|
+
job_type = 'S3_UNARCHIVING'
|
|
20
20
|
|
|
21
21
|
return fastq_unarchiving_post_request(
|
|
22
22
|
JOB_ENDPOINT,
|
|
23
23
|
params={
|
|
24
24
|
"fastqIds": fastq_ids,
|
|
25
|
-
"jobType": job_type
|
|
25
|
+
"jobType": job_type
|
|
26
26
|
}
|
|
27
27
|
)
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
|
|
3
3
|
import re
|
|
4
|
-
from enum import Enum
|
|
5
4
|
|
|
6
5
|
# AWS PARAMETERS
|
|
7
6
|
FASTQ_UNARCHIVING_SUBDOMAIN_NAME = "fastq-unarchiving"
|
|
@@ -12,10 +11,3 @@ JOB_ENDPOINT = "api/v1/jobs"
|
|
|
12
11
|
# REGEX
|
|
13
12
|
ORCABUS_ULID_REGEX_MATCH = re.compile(r'^[a-z0-9]{3}\.[A-Z0-9]{26}$')
|
|
14
13
|
|
|
15
|
-
|
|
16
|
-
class JobStatus(Enum):
|
|
17
|
-
PENDING = "PENDING"
|
|
18
|
-
RUNNING = "RUNNING"
|
|
19
|
-
FAILED = "FAILED"
|
|
20
|
-
ABORTED = "ABORTED"
|
|
21
|
-
SUCCEEDED = "SUCCEEDED"
|
|
@@ -13,28 +13,27 @@
|
|
|
13
13
|
|
|
14
14
|
from typing import (
|
|
15
15
|
TypedDict, NotRequired,
|
|
16
|
+
Literal
|
|
16
17
|
)
|
|
17
18
|
|
|
18
|
-
|
|
19
|
+
JobType = Literal[
|
|
20
|
+
"S3_UNARCHIVING",
|
|
21
|
+
]
|
|
19
22
|
|
|
20
23
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
FAILED = "FAILED"
|
|
29
|
-
ABORTED = "ABORTED"
|
|
30
|
-
SUCCEEDED = "SUCCEEDED"
|
|
31
|
-
|
|
24
|
+
JobStatusType = Literal[
|
|
25
|
+
"PENDING",
|
|
26
|
+
"RUNNING",
|
|
27
|
+
"FAILED",
|
|
28
|
+
"ABORTED",
|
|
29
|
+
"SUCCEEDED",
|
|
30
|
+
]
|
|
32
31
|
|
|
33
32
|
class Job(TypedDict):
|
|
34
33
|
id: str
|
|
35
34
|
jobType: JobType
|
|
36
35
|
stepsExecutionArn: str
|
|
37
|
-
status:
|
|
36
|
+
status: JobStatusType
|
|
38
37
|
startTime: str
|
|
39
38
|
endTime: str
|
|
40
39
|
errorMessages: str
|
|
@@ -42,7 +41,7 @@ class Job(TypedDict):
|
|
|
42
41
|
|
|
43
42
|
class JobQueryParameters(TypedDict):
|
|
44
43
|
fastqId: NotRequired[str]
|
|
45
|
-
status: NotRequired[
|
|
44
|
+
status: NotRequired[JobStatusType]
|
|
46
45
|
createdAfter: NotRequired[str]
|
|
47
46
|
createdBefore: NotRequired[str]
|
|
48
47
|
completedAfter: NotRequired[str]
|
package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq_unarchiving/query_helpers.py
CHANGED
|
@@ -14,7 +14,7 @@ from typing import List, Unpack
|
|
|
14
14
|
|
|
15
15
|
# Local imports
|
|
16
16
|
from . import get_fastq_unarchiving_request_response_results
|
|
17
|
-
from .models import Job,
|
|
17
|
+
from .models import Job, JobStatusType, JobQueryParameters
|
|
18
18
|
|
|
19
19
|
from .globals import JOB_ENDPOINT
|
|
20
20
|
|
|
@@ -40,7 +40,7 @@ def get_unarchiving_job_list(**kwargs: Unpack[JobQueryParameters]) -> List[Job]:
|
|
|
40
40
|
|
|
41
41
|
def get_job_list_for_fastq(
|
|
42
42
|
fastq_id: str,
|
|
43
|
-
job_status:
|
|
43
|
+
job_status: JobStatusType
|
|
44
44
|
) -> List[Job]:
|
|
45
45
|
"""
|
|
46
46
|
Check if fastq in job list
|
|
@@ -48,5 +48,5 @@ def get_job_list_for_fastq(
|
|
|
48
48
|
"""
|
|
49
49
|
return get_unarchiving_job_list(
|
|
50
50
|
fastqId=fastq_id,
|
|
51
|
-
status=job_status
|
|
51
|
+
status=job_status
|
|
52
52
|
)
|
package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq_unarchiving/update_helpers.py
CHANGED
|
@@ -17,13 +17,13 @@ from typing import Optional
|
|
|
17
17
|
|
|
18
18
|
# Local imports
|
|
19
19
|
from . import fastq_unarchiving_patch_request
|
|
20
|
-
from .globals import
|
|
21
|
-
from .models import Job
|
|
20
|
+
from .globals import JOB_ENDPOINT
|
|
21
|
+
from .models import Job, JobStatusType
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
def update_status(
|
|
25
25
|
job_id: str,
|
|
26
|
-
job_status:
|
|
26
|
+
job_status: JobStatusType,
|
|
27
27
|
error_message: Optional[str] = None
|
|
28
28
|
) -> Job:
|
|
29
29
|
"""
|
|
@@ -40,10 +40,12 @@ def get_file_manager_request(
|
|
|
40
40
|
|
|
41
41
|
def file_manager_patch_request(
|
|
42
42
|
endpoint: str,
|
|
43
|
+
json_data: Optional[Dict] = None,
|
|
43
44
|
params: Optional[Dict] = None
|
|
44
45
|
):
|
|
45
46
|
return patch_request(
|
|
46
47
|
get_file_manager_url(endpoint),
|
|
48
|
+
json_data=json_data,
|
|
47
49
|
params=params
|
|
48
50
|
)
|
|
49
51
|
|
|
@@ -1,24 +1,30 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
|
+
|
|
3
|
+
# Standard imports
|
|
4
|
+
import json
|
|
2
5
|
from functools import reduce
|
|
3
6
|
from operator import concat
|
|
4
7
|
from typing import List, Dict, Union
|
|
5
8
|
import typing
|
|
6
|
-
|
|
7
9
|
import boto3
|
|
10
|
+
from datetime import datetime, timedelta, timezone
|
|
11
|
+
from urllib.parse import urlparse
|
|
12
|
+
from itertools import batched
|
|
8
13
|
|
|
14
|
+
# Local imports
|
|
9
15
|
from .errors import S3FileNotFoundError, S3DuplicateFileCopyError
|
|
10
|
-
from .models import FileObject
|
|
16
|
+
from .models import FileObject, StorageClassPriority
|
|
11
17
|
from ..utils.miscell import get_bucket_key_pair_from_uri
|
|
12
|
-
from . import
|
|
18
|
+
from . import (
|
|
19
|
+
get_file_manager_request_response_results,
|
|
20
|
+
get_file_manager_request,
|
|
21
|
+
file_manager_patch_request
|
|
22
|
+
)
|
|
13
23
|
from .globals import (
|
|
14
24
|
S3_LIST_ENDPOINT,
|
|
15
25
|
S3_BUCKETS_BY_ACCOUNT_ID,
|
|
16
|
-
S3_PREFIXES_BY_ACCOUNT_ID
|
|
17
|
-
StorageEnum, StoragePriority,
|
|
26
|
+
S3_PREFIXES_BY_ACCOUNT_ID
|
|
18
27
|
)
|
|
19
|
-
from datetime import datetime, timedelta, timezone
|
|
20
|
-
from urllib.parse import urlparse
|
|
21
|
-
from itertools import batched
|
|
22
28
|
|
|
23
29
|
if typing.TYPE_CHECKING:
|
|
24
30
|
from mypy_boto3_sts import STSClient
|
|
@@ -94,7 +100,7 @@ def get_file_object_from_ingest_id(ingest_id: str, **kwargs) -> FileObject:
|
|
|
94
100
|
))
|
|
95
101
|
|
|
96
102
|
# Order by storage class
|
|
97
|
-
file_objects_list.sort(key=lambda file_obj_iter_:
|
|
103
|
+
file_objects_list.sort(key=lambda file_obj_iter_: StorageClassPriority[file_obj_iter_['storageClass']])
|
|
98
104
|
|
|
99
105
|
# Return as a FileObject model
|
|
100
106
|
return file_objects_list[0]
|
|
@@ -263,8 +269,9 @@ def get_s3_objs_from_ingest_ids_map(ingest_ids: List[str], **kwargs) -> List[Dic
|
|
|
263
269
|
continue
|
|
264
270
|
|
|
265
271
|
s3_objects_match.sort(
|
|
266
|
-
key=lambda s3_object_iter_:
|
|
267
|
-
|
|
272
|
+
key=lambda s3_object_iter_: StorageClassPriority[
|
|
273
|
+
s3_object_iter_['fileObject']['storageClass']
|
|
274
|
+
]
|
|
268
275
|
)
|
|
269
276
|
|
|
270
277
|
s3_objects_by_ingest_id_filtered.append(
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
#!/usr/bin/env python
|
|
2
|
-
from enum import Enum
|
|
3
2
|
|
|
4
3
|
# AWS PARAMETERS
|
|
5
4
|
FILEMANAGER_SUBDOMAIN_NAME = "file"
|
|
@@ -41,30 +40,3 @@ S3_PREFIXES_BY_ACCOUNT_ID = {
|
|
|
41
40
|
},
|
|
42
41
|
}
|
|
43
42
|
|
|
44
|
-
# FROM FileManager Schema
|
|
45
|
-
# "DeepArchive"
|
|
46
|
-
# "Glacier"
|
|
47
|
-
# "GlacierIr"
|
|
48
|
-
# "IntelligentTiering"
|
|
49
|
-
# "OnezoneIa"
|
|
50
|
-
# "Outposts"
|
|
51
|
-
# "ReducedRedundancy"
|
|
52
|
-
# "Snow"
|
|
53
|
-
# "Standard"
|
|
54
|
-
# "StandardIa"
|
|
55
|
-
class StorageEnum(Enum):
|
|
56
|
-
STANDARD = "Standard"
|
|
57
|
-
STANDARD_IA = "StandardIa"
|
|
58
|
-
INTELLIGENT_TIERING = "IntelligentTiering"
|
|
59
|
-
GLACIER_INSTANT_RETRIEVAL = "GlacierIr"
|
|
60
|
-
GLACIER = "Glacier"
|
|
61
|
-
DEEP_ARCHIVE = "DeepArchive"
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
class StoragePriority(Enum):
|
|
65
|
-
STANDARD = 1
|
|
66
|
-
STANDARD_IA = 2
|
|
67
|
-
INTELLIGENT_TIERING = 3
|
|
68
|
-
GLACIER_INSTANT_RETRIEVAL = 4
|
|
69
|
-
GLACIER = 5
|
|
70
|
-
DEEP_ARCHIVE = 6
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import Optional, TypedDict, Dict
|
|
1
|
+
from typing import Optional, TypedDict, Dict, Literal
|
|
2
2
|
|
|
3
3
|
"""
|
|
4
4
|
Example File Object response
|
|
@@ -26,6 +26,23 @@ Example File Object response
|
|
|
26
26
|
}
|
|
27
27
|
"""
|
|
28
28
|
|
|
29
|
+
StorageClassType = Literal[
|
|
30
|
+
"Standard",
|
|
31
|
+
"StandardIa",
|
|
32
|
+
"IntelligentTiering",
|
|
33
|
+
"GlacierIr",
|
|
34
|
+
"Glacier",
|
|
35
|
+
"DeepArchive",
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
StorageClassPriority: Dict[StorageClassType, int] = {
|
|
39
|
+
"Standard": 1,
|
|
40
|
+
"StandardIa": 2,
|
|
41
|
+
"IntelligentTiering": 3,
|
|
42
|
+
"GlacierIr": 4,
|
|
43
|
+
"Glacier": 5,
|
|
44
|
+
"DeepArchive": 6,
|
|
45
|
+
}
|
|
29
46
|
|
|
30
47
|
class FileObject(TypedDict):
|
|
31
48
|
# Identifier
|
|
@@ -47,7 +64,7 @@ class FileObject(TypedDict):
|
|
|
47
64
|
numberReordered: int
|
|
48
65
|
sequencer: str
|
|
49
66
|
size: int
|
|
50
|
-
storageClass:
|
|
67
|
+
storageClass: StorageClassType
|
|
51
68
|
|
|
52
69
|
# Attribute attributes
|
|
53
70
|
attributes: Optional[Dict]
|
|
@@ -9,6 +9,8 @@ from copy import deepcopy
|
|
|
9
9
|
|
|
10
10
|
from requests import HTTPError
|
|
11
11
|
|
|
12
|
+
from fastapi.encoders import jsonable_encoder
|
|
13
|
+
|
|
12
14
|
# Locals
|
|
13
15
|
from .aws_helpers import (
|
|
14
16
|
get_orcabus_token, get_hostname
|
|
@@ -103,7 +105,11 @@ def get_request(url: str, params: Optional[Dict] = None) -> Dict:
|
|
|
103
105
|
return response.json()
|
|
104
106
|
|
|
105
107
|
|
|
106
|
-
def patch_request(
|
|
108
|
+
def patch_request(
|
|
109
|
+
url: str,
|
|
110
|
+
json_data: Optional[Dict] = None,
|
|
111
|
+
params: Optional[Dict] = None
|
|
112
|
+
) -> Dict:
|
|
107
113
|
# Get authorization header
|
|
108
114
|
headers = {
|
|
109
115
|
"Authorization": f"Bearer {get_orcabus_token()}"
|
|
@@ -119,7 +125,8 @@ def patch_request(url: str, params: Optional[Dict] = None) -> Dict:
|
|
|
119
125
|
response = requests.patch(
|
|
120
126
|
url,
|
|
121
127
|
headers=headers,
|
|
122
|
-
|
|
128
|
+
params=req_params,
|
|
129
|
+
json=json_data
|
|
123
130
|
)
|
|
124
131
|
|
|
125
132
|
try:
|
package/monitored-queue/index.js
CHANGED
|
@@ -13,7 +13,7 @@ const aws_cloudwatch_actions_1 = require("aws-cdk-lib/aws-cloudwatch-actions");
|
|
|
13
13
|
* SNS topic.
|
|
14
14
|
*/
|
|
15
15
|
class MonitoredQueue extends constructs_1.Construct {
|
|
16
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.monitoredQueue.MonitoredQueue", version: "0.0.
|
|
16
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.monitoredQueue.MonitoredQueue", version: "0.0.35" };
|
|
17
17
|
queue;
|
|
18
18
|
deadLetterQueue;
|
|
19
19
|
alarm;
|
|
@@ -7,7 +7,7 @@ const aws_iam_1 = require("aws-cdk-lib/aws-iam");
|
|
|
7
7
|
* A construct which represents a named role that a Lambda function can assume.
|
|
8
8
|
*/
|
|
9
9
|
class NamedLambdaRole extends aws_iam_1.Role {
|
|
10
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.namedLambdaRole.NamedLambdaRole", version: "0.0.
|
|
10
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.namedLambdaRole.NamedLambdaRole", version: "0.0.35" };
|
|
11
11
|
constructor(scope, id, props) {
|
|
12
12
|
super(scope, id, {
|
|
13
13
|
assumedBy: new aws_iam_1.ServicePrincipal("lambda.amazonaws.com"),
|
package/package.json
CHANGED
|
@@ -15,7 +15,7 @@ const aws_cdk_lib_1 = require("aws-cdk-lib");
|
|
|
15
15
|
* to the provider framework.
|
|
16
16
|
*/
|
|
17
17
|
class ProviderFunction extends constructs_1.Construct {
|
|
18
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.provider.ProviderFunction", version: "0.0.
|
|
18
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.provider.ProviderFunction", version: "0.0.35" };
|
|
19
19
|
_function;
|
|
20
20
|
_response;
|
|
21
21
|
constructor(scope, id, props) {
|
|
@@ -23,7 +23,7 @@ exports.VPC_LOOKUP_PROPS = {
|
|
|
23
23
|
* Helper for looking up the shared OrcaBus VPC.
|
|
24
24
|
*/
|
|
25
25
|
class OrcaBusVpc {
|
|
26
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.sharedConfig.networking.OrcaBusVpc", version: "0.0.
|
|
26
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.sharedConfig.networking.OrcaBusVpc", version: "0.0.35" };
|
|
27
27
|
/**
|
|
28
28
|
* The shared VPC that is used by OrcaBus.
|
|
29
29
|
* @param scope
|
|
@@ -42,7 +42,7 @@ exports.SHARED_SECURITY_GROUP_NAME = "OrcaBusSharedComputeSecurityGroup";
|
|
|
42
42
|
* Helper for looking up the shared compute security group by name.
|
|
43
43
|
*/
|
|
44
44
|
class OrcaBusSharedComputeSecurityGroup {
|
|
45
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.sharedConfig.networking.OrcaBusSharedComputeSecurityGroup", version: "0.0.
|
|
45
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.sharedConfig.networking.OrcaBusSharedComputeSecurityGroup", version: "0.0.35" };
|
|
46
46
|
/**
|
|
47
47
|
* The shared security group that is used by compute resources to access the database.
|
|
48
48
|
* @param scope
|
package/shared-config/slack.js
CHANGED
|
@@ -11,7 +11,7 @@ exports.SLACK_ALERTS_SNS_TOPIC = "AwsChatBotTopic-alerts";
|
|
|
11
11
|
* A helper class to construct the arn for the slack SNS topic.
|
|
12
12
|
*/
|
|
13
13
|
class SlackAlerts {
|
|
14
|
-
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.sharedConfig.slack.SlackAlerts", version: "0.0.
|
|
14
|
+
static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.sharedConfig.slack.SlackAlerts", version: "0.0.35" };
|
|
15
15
|
/**
|
|
16
16
|
* Format the ARN for the slack alerts SNS topic for the current stack.
|
|
17
17
|
* @param stack
|