@orcabus/platform-cdk-constructs 0.0.34 → 0.0.35

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/.jsii +2 -2
  2. package/api-gateway/api-gateway.js +1 -1
  3. package/deployment-stack-pipeline/artifact-bucket.js +1 -1
  4. package/deployment-stack-pipeline/pipeline.js +1 -1
  5. package/dynamodb/index.js +2 -2
  6. package/ecs/index.js +1 -1
  7. package/lambda/index.js +1 -1
  8. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/data_sharing/__init__.py +70 -0
  9. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/data_sharing/globals.py +15 -0
  10. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/data_sharing/models.py +28 -0
  11. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/data_sharing/update_helpers.py +53 -0
  12. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/__init__.py +12 -2
  13. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/create_helpers.py +22 -5
  14. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/globals.py +1 -1
  15. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/job_helpers.py +16 -4
  16. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/models.py +81 -20
  17. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/query_helpers.py +25 -14
  18. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/update_helpers.py +42 -18
  19. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/workflow_helpers.py +3 -2
  20. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq_unarchiving/__init__.py +4 -1
  21. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq_unarchiving/create_helpers.py +2 -2
  22. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq_unarchiving/globals.py +0 -8
  23. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq_unarchiving/models.py +13 -14
  24. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq_unarchiving/query_helpers.py +3 -3
  25. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq_unarchiving/update_helpers.py +3 -3
  26. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/filemanager/__init__.py +2 -0
  27. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/filemanager/file_helpers.py +18 -11
  28. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/filemanager/globals.py +0 -28
  29. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/filemanager/models.py +19 -2
  30. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/utils/requests_helpers.py +9 -2
  31. package/monitored-queue/index.js +1 -1
  32. package/named-lambda-role/index.js +1 -1
  33. package/package.json +1 -1
  34. package/provider-function/index.js +1 -1
  35. package/shared-config/networking.js +2 -2
  36. package/shared-config/slack.js +1 -1
package/.jsii CHANGED
@@ -6756,6 +6756,6 @@
6756
6756
  "symbolId": "shared-config/slack:SlackAlerts"
6757
6757
  }
6758
6758
  },
6759
- "version": "0.0.34",
6760
- "fingerprint": "d+PMtKWVtmzS+P1omiV5f8tQJsCkfNLbBjjTeYDFmFM="
6759
+ "version": "0.0.35",
6760
+ "fingerprint": "dgr8YQrliJh0hWpEcsepSzBVhIznw5JYimaBU1p7VKg="
6761
6761
  }
@@ -15,7 +15,7 @@ const aws_lambda_1 = require("aws-cdk-lib/aws-lambda");
15
15
  const aws_route53_targets_1 = require("aws-cdk-lib/aws-route53-targets");
16
16
  const config_1 = require("./config");
17
17
  class OrcaBusApiGateway extends constructs_1.Construct {
18
- static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.apigateway.OrcaBusApiGateway", version: "0.0.34" };
18
+ static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.apigateway.OrcaBusApiGateway", version: "0.0.35" };
19
19
  /**
20
20
  * The AWS region where the API Gateway is deployed.
21
21
  */
@@ -13,7 +13,7 @@ exports.CROSS_DEPLOYMENT_ARTIFACT_BUCKET_NAME = "orcabus-cross-deployment-codepi
13
13
  exports.CROSS_DEPLOYMENT_ARTIFACT_KMS_ALIAS = "orcabus-cross-deployment-codepipeline-artifact";
14
14
  exports.CROSS_DEPLOYMENT_ARTIFACT_KMS_ARN_SSM_PARAMETER_NAME = "/orcabus/deployment-stack-pipeline/artifact-bucket/kms-key-arn";
15
15
  class CrossDeploymentArtifactBucket extends constructs_1.Construct {
16
- static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.deploymentPipeline.CrossDeploymentArtifactBucket", version: "0.0.34" };
16
+ static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.deploymentPipeline.CrossDeploymentArtifactBucket", version: "0.0.35" };
17
17
  /**
18
18
  * The S3 bucket used to store artifacts for cross-deployment pipelines.
19
19
  */
@@ -32,7 +32,7 @@ exports.DEFAULT_SYNTH_STEP_PARTIAL_BUILD_SPEC = {
32
32
  * before using this construct.
33
33
  */
34
34
  class DeploymentStackPipeline extends constructs_1.Construct {
35
- static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.deploymentPipeline.DeploymentStackPipeline", version: "0.0.34" };
35
+ static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.deploymentPipeline.DeploymentStackPipeline", version: "0.0.35" };
36
36
  /**
37
37
  * The code pipeline construct that is created.
38
38
  */
package/dynamodb/index.js CHANGED
@@ -40,7 +40,7 @@ const dynamodb = __importStar(require("aws-cdk-lib/aws-dynamodb"));
40
40
  const aws_cdk_lib_1 = require("aws-cdk-lib");
41
41
  const config_1 = require("./config");
42
42
  class DynamoDbPartitionedConstruct extends constructs_1.Construct {
43
- static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.dynamodb.DynamoDbPartitionedConstruct", version: "0.0.34" };
43
+ static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.dynamodb.DynamoDbPartitionedConstruct", version: "0.0.35" };
44
44
  table;
45
45
  constructor(scope, id, props) {
46
46
  super(scope, id);
@@ -75,7 +75,7 @@ class DynamoDbPartitionedConstruct extends constructs_1.Construct {
75
75
  }
76
76
  exports.DynamoDbPartitionedConstruct = DynamoDbPartitionedConstruct;
77
77
  class DynamoDbNonPartitionedConstruct extends constructs_1.Construct {
78
- static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.dynamodb.DynamoDbNonPartitionedConstruct", version: "0.0.34" };
78
+ static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.dynamodb.DynamoDbNonPartitionedConstruct", version: "0.0.35" };
79
79
  table;
80
80
  constructor(scope, id, props) {
81
81
  super(scope, id);
package/ecs/index.js CHANGED
@@ -55,7 +55,7 @@ exports.LAMBDA_ARCHITECTURE_MAP = {
55
55
  ['ARM64']: lambda.Architecture.ARM_64
56
56
  };
57
57
  class EcsFargateTaskConstruct extends constructs_1.Construct {
58
- static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.ecs.EcsFargateTaskConstruct", version: "0.0.34" };
58
+ static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.ecs.EcsFargateTaskConstruct", version: "0.0.35" };
59
59
  cluster;
60
60
  taskDefinition;
61
61
  taskExecutionRole;
package/lambda/index.js CHANGED
@@ -56,7 +56,7 @@ function getPythonUvDockerImage() {
56
56
  return aws_cdk_lib_1.DockerImage.fromBuild(path_1.default.join(__dirname, 'build_python'));
57
57
  }
58
58
  class PythonUvFunction extends aws_lambda_python_alpha_1.PythonFunction {
59
- static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.lambda.PythonUvFunction", version: "0.0.34" };
59
+ static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.lambda.PythonUvFunction", version: "0.0.35" };
60
60
  // Class constructs, to be used for caching the layers
61
61
  // This means that if there are multiple lambdas throughout the stack
62
62
  // They will all use the same layer
@@ -0,0 +1,70 @@
1
+ # FIMXE Implement the following imports
2
+ from typing import Optional, Dict
3
+
4
+ from .globals import DATA_SHARING_SUBDOMAIN_NAME
5
+ from ..utils.requests_helpers import get_url, get_request, get_request_response_results, patch_request
6
+
7
+
8
+ # Get url for the subdomain
9
+ def get_data_sharing_url(endpoint: str) -> str:
10
+ """
11
+ Get the URL for the data sharing endpoint
12
+ :param endpoint:
13
+ :return:
14
+ """
15
+ return get_url(
16
+ endpoint=endpoint,
17
+ subdomain=DATA_SHARING_SUBDOMAIN_NAME,
18
+ )
19
+
20
+
21
+ # Wrappers
22
+ def get_data_sharing_request(
23
+ endpoint: str,
24
+ params: Optional[Dict] = None,
25
+ ):
26
+ return get_request(
27
+ url=get_data_sharing_url(endpoint),
28
+ params=params
29
+ )
30
+
31
+
32
+ def get_data_sharing_request_response_results(
33
+ endpoint: str,
34
+ params: Optional[Dict] = None,
35
+ ):
36
+ return get_request_response_results(
37
+ url=get_data_sharing_url(endpoint),
38
+ params=params
39
+ )
40
+
41
+
42
+ def data_sharing_patch_request(
43
+ endpoint: str,
44
+ params: Optional[Dict] = None,
45
+ ):
46
+ return patch_request(
47
+ url=get_data_sharing_url(endpoint),
48
+ params=params
49
+ )
50
+
51
+
52
+ def data_sharing_post_request(
53
+ endpoint: str,
54
+ params: Optional[Dict] = None,
55
+ ):
56
+ return patch_request(
57
+ url=get_data_sharing_url(endpoint),
58
+ params=params
59
+ )
60
+
61
+
62
+ from .update_helpers import (
63
+ update_package_status,
64
+ update_push_job_status,
65
+ )
66
+
67
+ __all__ = [
68
+ "update_package_status",
69
+ "update_push_job_status",
70
+ ]
@@ -0,0 +1,15 @@
1
+ # Imports
2
+ import re
3
+ from typing import Literal
4
+
5
+ # AWS PARAMETERS
6
+ DATA_SHARING_SUBDOMAIN_NAME = "data-sharing"
7
+
8
+ # API ENDPOINTS
9
+ PACKAGE_ENDPOINT = "api/v1/package"
10
+ PUSH_ENDPOINT = "api/v1/push"
11
+
12
+ # REGEX
13
+ ORCABUS_ULID_REGEX_MATCH = re.compile(r'^[a-z0-9]{3}\.[A-Z0-9]{26}$')
14
+
15
+
@@ -0,0 +1,28 @@
1
+ from typing import Literal, TypedDict, NotRequired
2
+
3
+ PackageStatusType = Literal[
4
+ "PENDING",
5
+ "RUNNING",
6
+ "FAILED",
7
+ "ABORTED",
8
+ "SUCCEEDED",
9
+ ]
10
+
11
+ PushJobStatusType = Literal[
12
+ "PENDING",
13
+ "RUNNING",
14
+ "FAILED",
15
+ "ABORTED",
16
+ "SUCCEEDED",
17
+ ]
18
+
19
+
20
+
21
+ class PackageObject(TypedDict):
22
+ id: str
23
+ packageName: str
24
+ stepsExecutionArn: str
25
+ status: PackageStatusType
26
+ requestTime: str
27
+ completionTime: NotRequired[str]
28
+ hasExpired: NotRequired[bool]
@@ -0,0 +1,53 @@
1
+ from typing import Optional
2
+
3
+ from . import data_sharing_patch_request
4
+ from .globals import PACKAGE_ENDPOINT, PUSH_ENDPOINT
5
+ from .models import PackageStatusType, PackageObject, PushJobStatusType
6
+
7
+
8
+ def update_package_status(
9
+ package_id: str,
10
+ package_status: PackageStatusType,
11
+ error_message: Optional[str] = None
12
+ ) -> PackageObject:
13
+ """
14
+ Add QC stats to a fastq_id.
15
+
16
+ :param package_id: The package id
17
+ :param package_status: The package status to set
18
+ :param error_message: Optional error message
19
+ """
20
+ return data_sharing_patch_request(
21
+ f"{PACKAGE_ENDPOINT}/{package_id}",
22
+ params=dict(filter(
23
+ lambda x: x[1] is not None,
24
+ {
25
+ "status": package_status,
26
+ "error_message": error_message
27
+ }.items()
28
+ ))
29
+ )
30
+
31
+
32
+ def update_push_job_status(
33
+ push_job_id: str,
34
+ push_job_status: PushJobStatusType,
35
+ error_message: Optional[str] = None
36
+ ) -> PackageObject:
37
+ """
38
+ Add push job status to a push job.
39
+
40
+ :param push_job_id: The push job id
41
+ :param push_job_status: The push job status to set
42
+ :param error_message: Optional error message
43
+ """
44
+ return data_sharing_patch_request(
45
+ f"{PUSH_ENDPOINT}/{push_job_id}",
46
+ params=dict(filter(
47
+ lambda x: x[1] is not None,
48
+ {
49
+ "status": push_job_status,
50
+ "error_message": error_message
51
+ }.items()
52
+ ))
53
+ )
@@ -70,14 +70,16 @@ def fastq_post_request(
70
70
  # Create
71
71
  from .create_helpers import (
72
72
  create_fastq_set_object,
73
- create_fastq_list_row_object
73
+ create_fastq_list_row_object,
74
+ create_fastq_object,
74
75
  )
75
76
 
76
77
  # Job
77
78
  from .job_helpers import (
78
79
  run_qc_stats,
79
80
  run_ntsm,
80
- run_file_compression_stats
81
+ run_file_compression_stats,
82
+ run_read_count_stats
81
83
  )
82
84
 
83
85
  # Query
@@ -97,6 +99,7 @@ from .query_helpers import (
97
99
  get_fastqs_in_individual,
98
100
  get_fastqs_in_project,
99
101
  get_fastq_list_rows_in_fastq_set,
102
+ get_fastqs_in_fastq_set,
100
103
  get_fastq_jobs,
101
104
  )
102
105
 
@@ -111,7 +114,9 @@ from .update_helpers import (
111
114
  validate_fastq,
112
115
  invalidate_fastq,
113
116
  link_fastq_list_row_to_fastq_set,
117
+ link_fastq_to_fastq_set,
114
118
  unlink_fastq_list_row_from_fastq_set,
119
+ unlink_fastq_from_fastq_set,
115
120
  allow_additional_fastqs_to_fastq_set,
116
121
  disallow_additional_fastqs_to_fastq_set,
117
122
  set_is_current_fastq_set,
@@ -135,10 +140,12 @@ __all__ = [
135
140
  # Create
136
141
  "create_fastq_set_object",
137
142
  "create_fastq_list_row_object",
143
+ "create_fastq_object",
138
144
  # Job
139
145
  "run_qc_stats",
140
146
  "run_ntsm",
141
147
  "run_file_compression_stats",
148
+ "run_read_count_stats",
142
149
  # Query
143
150
  "get_fastq",
144
151
  "get_fastq_set",
@@ -155,6 +162,7 @@ __all__ = [
155
162
  "get_fastqs_in_individual",
156
163
  "get_fastqs_in_project",
157
164
  "get_fastq_list_rows_in_fastq_set",
165
+ "get_fastqs_in_fastq_set",
158
166
  "get_fastq_jobs",
159
167
  # Updaters
160
168
  "add_qc_stats",
@@ -166,7 +174,9 @@ __all__ = [
166
174
  "validate_fastq",
167
175
  "invalidate_fastq",
168
176
  "link_fastq_list_row_to_fastq_set",
177
+ "link_fastq_to_fastq_set",
169
178
  "unlink_fastq_list_row_from_fastq_set",
179
+ "unlink_fastq_from_fastq_set",
170
180
  "allow_additional_fastqs_to_fastq_set",
171
181
  "disallow_additional_fastqs_to_fastq_set",
172
182
  "set_is_current_fastq_set",
@@ -17,18 +17,30 @@ from typing import Unpack
17
17
 
18
18
  # Local imports
19
19
  from . import fastq_post_request
20
- from .globals import FASTQ_LIST_ROW_ENDPOINT, FASTQ_SET_ENDPOINT
21
- from .models import FastqListRow, FastqSet, FastqListRowCreate, FastqSetCreate
20
+ from .globals import FASTQ_ENDPOINT, FASTQ_SET_ENDPOINT
21
+ from .models import Fastq, FastqSet, FastqCreate, FastqSetCreate
22
22
 
23
23
 
24
- def create_fastq_list_row_object(**kwargs: Unpack[FastqListRowCreate]) -> FastqListRow:
24
+ def create_fastq_list_row_object(**kwargs: Unpack[FastqCreate]) -> Fastq:
25
+ """
26
+ DEPRECATED: Use create_fastq_object instead.
27
+ """
28
+ return create_fastq_object(**kwargs)
29
+
30
+
31
+ def create_fastq_object(**kwargs: Unpack[FastqCreate]) -> Fastq:
25
32
  """
26
33
  Add a fastq list row object to the database.
27
34
  Returns the created fastq list row object
28
35
  """
29
- return FastqListRow(
36
+ # Raise error if any of the kwargs are not in the FastqCreate
37
+ for key in kwargs.keys():
38
+ if key not in FastqCreate.__annotations__:
39
+ raise ValueError(f"Invalid parameter: {key}")
40
+
41
+ return Fastq(
30
42
  **fastq_post_request(
31
- endpoint=FASTQ_LIST_ROW_ENDPOINT,
43
+ endpoint=FASTQ_ENDPOINT,
32
44
  params=dict(kwargs)
33
45
  )
34
46
  )
@@ -39,6 +51,11 @@ def create_fastq_set_object(**kwargs: Unpack[FastqSetCreate]) -> FastqSet:
39
51
  Add a fastq set object to the database.
40
52
  Returns the created fastq set object
41
53
  """
54
+ # Raise error if any of the kwargs are not in the FastqCreate
55
+ for key in kwargs.keys():
56
+ if key not in FastqSetCreate.__annotations__:
57
+ raise ValueError(f"Invalid parameter: {key}")
58
+
42
59
  return FastqSet(
43
60
  **fastq_post_request(
44
61
  endpoint=FASTQ_SET_ENDPOINT,
@@ -6,7 +6,7 @@ import re
6
6
  FASTQ_SUBDOMAIN_NAME = "fastq"
7
7
 
8
8
  # API ENDPOINTS
9
- FASTQ_LIST_ROW_ENDPOINT = "api/v1/fastq"
9
+ FASTQ_ENDPOINT = "api/v1/fastq"
10
10
  FASTQ_SET_ENDPOINT = "api/v1/fastqSet"
11
11
 
12
12
  # REGEX
@@ -11,7 +11,7 @@ Update helpers for the update script.
11
11
 
12
12
  # Local imports
13
13
  from . import fastq_patch_request
14
- from .globals import FASTQ_LIST_ROW_ENDPOINT
14
+ from .globals import FASTQ_ENDPOINT
15
15
  from .models import Job
16
16
 
17
17
 
@@ -23,7 +23,7 @@ def run_qc_stats(fastq_id: str) -> Job:
23
23
  """
24
24
  return Job(
25
25
  **fastq_patch_request(
26
- f"{FASTQ_LIST_ROW_ENDPOINT}/{fastq_id}:runQcStats"
26
+ f"{FASTQ_ENDPOINT}/{fastq_id}:runQcStats"
27
27
  )
28
28
  )
29
29
 
@@ -36,7 +36,7 @@ def run_ntsm(fastq_id: str) -> Job:
36
36
  """
37
37
  return Job(
38
38
  **fastq_patch_request(
39
- f"{FASTQ_LIST_ROW_ENDPOINT}/{fastq_id}:runNtsm"
39
+ f"{FASTQ_ENDPOINT}/{fastq_id}:runNtsm"
40
40
  )
41
41
  )
42
42
 
@@ -48,6 +48,18 @@ def run_file_compression_stats(fastq_id: str) -> Job:
48
48
  """
49
49
  return Job(
50
50
  **fastq_patch_request(
51
- f"{FASTQ_LIST_ROW_ENDPOINT}/{fastq_id}:runFileCompressionInformation"
51
+ f"{FASTQ_ENDPOINT}/{fastq_id}:runFileCompressionInformation"
52
+ )
53
+ )
54
+
55
+ def run_read_count_stats(fastq_id: str) -> Job:
56
+ """
57
+ Run file compression stats for a fastq_id.
58
+
59
+ :param fastq_id: Fastq str
60
+ """
61
+ return Job(
62
+ **fastq_patch_request(
63
+ f"{FASTQ_ENDPOINT}/{fastq_id}:runReadCountInformation"
52
64
  )
53
65
  )
@@ -40,22 +40,27 @@ from typing import (
40
40
  TypedDict,
41
41
  Optional,
42
42
  Dict,
43
- List, NotRequired, Union, Literal
43
+ List, NotRequired, Union,
44
+ Literal
44
45
  )
45
46
  from datetime import datetime
46
47
 
47
48
 
48
- class JobType(Enum):
49
- QC = 'QC'
50
- FILE_COMPRESSION = 'FILE_COMPRESSION'
51
- NTSM = 'NTSM'
49
+ JobType = Literal[
50
+ 'QC',
51
+ 'FILE_COMPRESSION',
52
+ 'NTSM',
53
+ 'READ_COUNT',
54
+ ]
55
+
52
56
 
57
+ JobStatus = Literal[
58
+ "PENDING",
59
+ "RUNNING",
60
+ "FAILED",
61
+ "SUCCEEDED",
62
+ ]
53
63
 
54
- class JobStatus(Enum):
55
- PENDING = "PENDING"
56
- RUNNING = "RUNNING"
57
- FAILED = "FAILED"
58
- SUCCEEDED = "SUCCEEDED"
59
64
 
60
65
 
61
66
  class FileStorageObject(TypedDict):
@@ -81,6 +86,7 @@ class Library(TypedDict):
81
86
  libraryId: str
82
87
 
83
88
 
89
+ # Deprecated: Use FastqCreate instead
84
90
  class FastqListRowCreate(TypedDict):
85
91
  fastqSetId: Optional[str]
86
92
  index: str
@@ -98,6 +104,7 @@ class FastqListRowCreate(TypedDict):
98
104
  ntsm: Optional[FileStorageObject]
99
105
 
100
106
 
107
+ # Deprecated: Use Fastq instead
101
108
  class FastqListRow(TypedDict):
102
109
  id: str
103
110
  fastqSetId: Optional[str]
@@ -116,9 +123,44 @@ class FastqListRow(TypedDict):
116
123
  ntsm: Optional[FileStorageObject]
117
124
 
118
125
 
126
+ class Fastq(TypedDict):
127
+ id: str
128
+ fastqSetId: Optional[str]
129
+ index: str
130
+ lane: int
131
+ instrumentRunId: str
132
+ library: Library
133
+ platform: Optional[str]
134
+ center: Optional[str]
135
+ date: Optional[datetime]
136
+ readSet: Optional[ReadSet]
137
+ qc: Optional[Dict]
138
+ readCount: Optional[int]
139
+ baseCountEst: Optional[int]
140
+ isValid: Optional[bool]
141
+ ntsm: Optional[FileStorageObject]
142
+
143
+
144
+ class FastqCreate(TypedDict):
145
+ fastqSetId: Optional[str]
146
+ index: str
147
+ lane: int
148
+ instrumentRunId: str
149
+ library: Library
150
+ platform: Optional[str]
151
+ center: Optional[str]
152
+ date: Optional[datetime]
153
+ readSet: Optional[ReadSet]
154
+ qc: Optional[Dict]
155
+ readCount: Optional[int]
156
+ baseCountEst: Optional[int]
157
+ isValid: Optional[bool]
158
+ ntsm: Optional[FileStorageObject]
159
+
160
+
119
161
  class FastqSetCreate(TypedDict):
120
162
  library: Library
121
- fastqSet: List[Union[str, FastqListRow]]
163
+ fastqSet: List[Union[str, Fastq]]
122
164
  allowAdditionalFastq: bool
123
165
  isCurrentFastqSet: bool
124
166
 
@@ -126,7 +168,7 @@ class FastqSetCreate(TypedDict):
126
168
  class FastqSet(TypedDict):
127
169
  id: str
128
170
  library: Library
129
- fastqSet: List[FastqListRow]
171
+ fastqSet: List[Fastq]
130
172
  allowAdditionalFastq: bool
131
173
  isCurrentFastqSet: bool
132
174
 
@@ -176,15 +218,21 @@ class Job(TypedDict):
176
218
  startTime: datetime
177
219
  endTime: Optional[datetime]
178
220
 
179
-
221
+ # Deprecated: Use BoolLiteral instead
180
222
  class BoolAllEnum(Enum):
181
223
  ALL = "ALL"
182
224
  true = True
183
225
  false = False
184
226
 
227
+ BoolLiteral = Literal[
228
+ 'ALL',
229
+ True,
230
+ False
231
+ ]
232
+
185
233
 
186
234
  class FastqGetResponseParameters(TypedDict):
187
- includeS3Details: NotRequired[BoolAllEnum]
235
+ includeS3Details: NotRequired[BoolLiteral]
188
236
 
189
237
 
190
238
  class StandardQueryParameters(TypedDict):
@@ -224,15 +272,28 @@ InstrumentRunIdQueryParametersList = TypedDict(
224
272
  )
225
273
 
226
274
 
227
- class FastqListRowQueryParameters(
275
+ # Deprecated: Use FastqQueryParameters instead
276
+ class FastqParameters(
277
+ StandardQueryParameters,
278
+ MetadataQueryParameter,
279
+ InstrumentRunIdQueryParameters,
280
+ MetadataQueryParametersList,
281
+ InstrumentRunIdQueryParametersList
282
+ ):
283
+ valid: NotRequired[BoolLiteral]
284
+ includeS3Details: NotRequired[BoolLiteral]
285
+ fastqSetId: NotRequired[str]
286
+
287
+
288
+ class FastqQueryParameters(
228
289
  StandardQueryParameters,
229
290
  MetadataQueryParameter,
230
291
  InstrumentRunIdQueryParameters,
231
292
  MetadataQueryParametersList,
232
293
  InstrumentRunIdQueryParametersList
233
294
  ):
234
- valid: NotRequired[BoolAllEnum]
235
- includeS3Details: NotRequired[BoolAllEnum]
295
+ valid: NotRequired[BoolLiteral]
296
+ includeS3Details: NotRequired[BoolLiteral]
236
297
  fastqSetId: NotRequired[str]
237
298
 
238
299
 
@@ -242,9 +303,9 @@ class FastqSetQueryParameters(
242
303
  MetadataQueryParametersList,
243
304
  InstrumentRunIdQueryParameters
244
305
  ):
245
- currentFastqSet: NotRequired[BoolAllEnum]
246
- allowAdditionalFastq: NotRequired[BoolAllEnum]
247
- includeS3Details: NotRequired[BoolAllEnum]
306
+ currentFastqSet: NotRequired[BoolLiteral]
307
+ allowAdditionalFastq: NotRequired[BoolLiteral]
308
+ includeS3Details: NotRequired[BoolLiteral]
248
309
 
249
310
  # Additional types
250
311
  VALID_BATCH_KEYS = Literal[
@@ -25,20 +25,22 @@ from typing import List, Unpack
25
25
  from fastapi.encoders import jsonable_encoder
26
26
 
27
27
  from . import get_fastq_request_response_results, get_fastq_request
28
- from .globals import FASTQ_LIST_ROW_ENDPOINT, FASTQ_SET_ENDPOINT
29
- from .models import FastqListRow, FastqSet, Job, FastqListRowQueryParameters, FastqSetQueryParameters, \
30
- FastqGetResponseParameters, VALID_BATCH_KEYS
28
+ from .globals import FASTQ_ENDPOINT, FASTQ_SET_ENDPOINT
29
+ from .models import (
30
+ FastqSet, Job, FastqParameters, FastqSetQueryParameters,
31
+ FastqGetResponseParameters, VALID_BATCH_KEYS, Fastq
32
+ )
31
33
 
32
34
 
33
- def get_fastq(fastq_id: str, **kwargs: Unpack[FastqGetResponseParameters]) -> FastqListRow:
35
+ def get_fastq(fastq_id: str, **kwargs: Unpack[FastqGetResponseParameters]) -> Fastq:
34
36
  # Raise error if any of the kwargs are not in the FastqSetQueryParameters
35
37
  for key in kwargs.keys():
36
38
  if key not in FastqGetResponseParameters.__annotations__:
37
39
  raise ValueError(f"Invalid parameter: {key}")
38
40
 
39
- return FastqListRow(
41
+ return Fastq(
40
42
  **get_fastq_request(
41
- f"{FASTQ_LIST_ROW_ENDPOINT}/{fastq_id}",
43
+ f"{FASTQ_ENDPOINT}/{fastq_id}",
42
44
  params=dict(kwargs)
43
45
  )
44
46
  )
@@ -67,17 +69,17 @@ def get_fastq_set(
67
69
  )
68
70
 
69
71
 
70
- def get_fastqs(**kwargs: Unpack[FastqListRowQueryParameters]) -> List[FastqListRow]:
72
+ def get_fastqs(**kwargs: Unpack[FastqParameters]) -> List[Fastq]:
71
73
  """
72
74
  Get all fastqs
73
75
  """
74
- # Raise error if any of the kwargs are not in the FastqListRowQueryParameters
76
+ # Raise error if any of the kwargs are not in the FastqParameters
75
77
  for key in kwargs.keys():
76
- if key not in FastqListRowQueryParameters.__annotations__:
78
+ if key not in FastqParameters.__annotations__:
77
79
  raise ValueError(f"Invalid parameter: {key}")
78
80
 
79
81
  return get_fastq_request_response_results(
80
- FASTQ_LIST_ROW_ENDPOINT,
82
+ FASTQ_ENDPOINT,
81
83
  params=dict(kwargs)
82
84
  )
83
85
 
@@ -89,7 +91,7 @@ def get_fastq_sets(**kwargs: Unpack[FastqSetQueryParameters]) -> List[FastqSet]:
89
91
  :param kwargs:
90
92
  :return:
91
93
  """
92
- # Raise error if any of the kwargs are not in the FastqListRowQueryParameters
94
+ # Raise error if any of the kwargs are not in the FastqParameters
93
95
  for key in kwargs.keys():
94
96
  if key not in FastqSetQueryParameters.__annotations__:
95
97
  raise ValueError(f"Invalid parameter: {key}")
@@ -123,7 +125,7 @@ def get_fastqs_batched(
123
125
  item_list: List[str],
124
126
  batch_size: int = 100,
125
127
  **kwargs
126
- ) -> List[FastqListRow]:
128
+ ) -> List[Fastq]:
127
129
  """
128
130
  Get all fastqs in a list of libraries
129
131
  """
@@ -153,7 +155,7 @@ def get_fastqs_batched(
153
155
 
154
156
  def get_fastqs_in_library_list(
155
157
  library_id_list: List[str]
156
- ) -> List[FastqListRow]:
158
+ ) -> List[Fastq]:
157
159
  """
158
160
  Get all fastqs in a list of libraries
159
161
  """
@@ -229,8 +231,17 @@ def get_fastqs_in_project(project_id):
229
231
 
230
232
 
231
233
  def get_fastq_list_rows_in_fastq_set(fastq_set_id):
234
+ """
235
+ DEPRECATED: Use get_fastqs_in_fastq_set instead
236
+ """
237
+ return get_fastqs_in_fastq_set(fastq_set_id)
238
+
239
+
240
+ def get_fastqs_in_fastq_set(fastq_set_id: str) -> List[Fastq]:
232
241
  """
233
242
  Get all fastqs in a fastq set
243
+ :param fastq_set_id:
244
+ :return:
234
245
  """
235
246
  return get_fastqs(
236
247
  fastqSetId=fastq_set_id
@@ -244,6 +255,6 @@ def get_fastq_jobs(fastq_id: str) -> List[Job]:
244
255
  return list(map(
245
256
  lambda job_iter_: Job(**job_iter_),
246
257
  get_fastq_request_response_results(
247
- f"{FASTQ_LIST_ROW_ENDPOINT}/{fastq_id}/jobs"
258
+ f"{FASTQ_ENDPOINT}/{fastq_id}/jobs"
248
259
  )
249
260
  ))
@@ -15,15 +15,15 @@ Update helpers for the update script.
15
15
 
16
16
  # Local imports
17
17
  from . import fastq_patch_request
18
- from .globals import FASTQ_LIST_ROW_ENDPOINT, FASTQ_SET_ENDPOINT
18
+ from .globals import FASTQ_ENDPOINT, FASTQ_SET_ENDPOINT
19
19
  from .models import (
20
- QcStats, FastqListRow, ReadCount,
20
+ QcStats, Fastq, ReadCount,
21
21
  FileCompressionInformation, FileStorageObject, ReadSet,
22
22
  FastqSet
23
23
  )
24
24
 
25
25
 
26
- def add_qc_stats(fastq_id: str, qc_stats: QcStats) -> FastqListRow:
26
+ def add_qc_stats(fastq_id: str, qc_stats: QcStats) -> Fastq:
27
27
  """
28
28
  Add QC stats to a fastq_id.
29
29
 
@@ -36,12 +36,12 @@ def add_qc_stats(fastq_id: str, qc_stats: QcStats) -> FastqListRow:
36
36
  raise ValueError(f"Invalid parameter: {key}")
37
37
 
38
38
  return fastq_patch_request(
39
- f"{FASTQ_LIST_ROW_ENDPOINT}/{fastq_id}/addQcStats",
39
+ f"{FASTQ_ENDPOINT}/{fastq_id}/addQcStats",
40
40
  params=dict(qc_stats)
41
41
  )
42
42
 
43
43
 
44
- def add_read_count(fastq_id: str, read_count: ReadCount) -> FastqListRow:
44
+ def add_read_count(fastq_id: str, read_count: ReadCount) -> Fastq:
45
45
  """
46
46
  Add read count to a fastq id
47
47
  :param fastq_id:
@@ -53,12 +53,12 @@ def add_read_count(fastq_id: str, read_count: ReadCount) -> FastqListRow:
53
53
  raise ValueError(f"Invalid parameter: {key}")
54
54
 
55
55
  return fastq_patch_request(
56
- f"{FASTQ_LIST_ROW_ENDPOINT}/{fastq_id}/addReadCount",
56
+ f"{FASTQ_ENDPOINT}/{fastq_id}/addReadCount",
57
57
  params=dict(read_count)
58
58
  )
59
59
 
60
60
 
61
- def add_file_compression_information(fastq_id: str, file_compression_information: FileCompressionInformation) -> FastqListRow:
61
+ def add_file_compression_information(fastq_id: str, file_compression_information: FileCompressionInformation) -> Fastq:
62
62
  """
63
63
  Add file compression information to a fastq id
64
64
  :param fastq_id:
@@ -70,12 +70,12 @@ def add_file_compression_information(fastq_id: str, file_compression_information
70
70
  raise ValueError(f"Invalid parameter: {key}")
71
71
 
72
72
  return fastq_patch_request(
73
- f"{FASTQ_LIST_ROW_ENDPOINT}/{fastq_id}/addFileCompressionInformation",
73
+ f"{FASTQ_ENDPOINT}/{fastq_id}/addFileCompressionInformation",
74
74
  params=dict(file_compression_information)
75
75
  )
76
76
 
77
77
 
78
- def add_ntsm_storage_object(fastq_id: str, ntsm_fastq_storage_object: FileStorageObject) -> FastqListRow:
78
+ def add_ntsm_storage_object(fastq_id: str, ntsm_fastq_storage_object: FileStorageObject) -> Fastq:
79
79
  """
80
80
  Add a Ntsm storage object to a fastq id.
81
81
 
@@ -87,12 +87,12 @@ def add_ntsm_storage_object(fastq_id: str, ntsm_fastq_storage_object: FileStorag
87
87
  raise ValueError(f"Invalid parameter: {key}")
88
88
 
89
89
  return fastq_patch_request(
90
- f"{FASTQ_LIST_ROW_ENDPOINT}/{fastq_id}/addNtsmStorageObject",
90
+ f"{FASTQ_ENDPOINT}/{fastq_id}/addNtsmStorageObject",
91
91
  params=dict(ntsm_fastq_storage_object)
92
92
  )
93
93
 
94
94
 
95
- def add_read_set(fastq_id: str, read_set: ReadSet) -> FastqListRow:
95
+ def add_read_set(fastq_id: str, read_set: ReadSet) -> Fastq:
96
96
  """
97
97
  Add a read set to a fastq id.
98
98
 
@@ -104,48 +104,62 @@ def add_read_set(fastq_id: str, read_set: ReadSet) -> FastqListRow:
104
104
  raise ValueError(f"Invalid parameter: {key}")
105
105
 
106
106
  return fastq_patch_request(
107
- f"{FASTQ_LIST_ROW_ENDPOINT}/{fastq_id}/addFastqPairStorageObject",
107
+ f"{FASTQ_ENDPOINT}/{fastq_id}/addFastqPairStorageObject",
108
108
  params=dict(read_set)
109
109
  )
110
110
 
111
111
 
112
- def detach_read_set(fastq_id: str) -> FastqListRow:
112
+ def detach_read_set(fastq_id: str) -> Fastq:
113
113
  """
114
114
  Detach a read set to a fastq id.
115
115
 
116
116
  :param fastq_id: Fastq str
117
117
  """
118
118
  return fastq_patch_request(
119
- f"{FASTQ_LIST_ROW_ENDPOINT}/{fastq_id}/detachFastqPairStorageObject"
119
+ f"{FASTQ_ENDPOINT}/{fastq_id}/detachFastqPairStorageObject"
120
120
  )
121
121
 
122
122
 
123
- def validate_fastq(fastq_id: str) -> FastqListRow:
123
+ def validate_fastq(fastq_id: str) -> Fastq:
124
124
  """
125
125
  Validate a fastq id.
126
126
 
127
127
  :param fastq_id: Fastq str
128
128
  """
129
129
  return fastq_patch_request(
130
- f"{FASTQ_LIST_ROW_ENDPOINT}/{fastq_id}/validate"
130
+ f"{FASTQ_ENDPOINT}/{fastq_id}/validate"
131
131
  )
132
132
 
133
133
 
134
- def invalidate_fastq(fastq_id: str) -> FastqListRow:
134
+ def invalidate_fastq(fastq_id: str) -> Fastq:
135
135
  """
136
136
  Invalidate a fastq id.
137
137
 
138
138
  :param fastq_id: Fastq str
139
139
  """
140
140
  return fastq_patch_request(
141
- f"{FASTQ_LIST_ROW_ENDPOINT}/{fastq_id}/invalidate"
141
+ f"{FASTQ_ENDPOINT}/{fastq_id}/invalidate"
142
142
  )
143
143
 
144
144
 
145
145
  def link_fastq_list_row_to_fastq_set(fastq_id: str, fastq_set_id: str) -> FastqSet:
146
146
  """
147
+ Deprecated: Use `link_fastq_to_fast_set` instead.
147
148
  Link a fastq id to a fastq set.
148
149
 
150
+ :param fastq_id:
151
+ :param fastq_set_id:
152
+ :return:
153
+ """
154
+ return link_fastq_to_fastq_set(
155
+ fastq_id=fastq_id,
156
+ fastq_set_id=fastq_set_id
157
+ )
158
+
159
+
160
+ def link_fastq_to_fastq_set(fastq_id: str, fastq_set_id: str) -> FastqSet:
161
+ """
162
+ Link a fastq id to a fastq set.
149
163
  :param fastq_id:
150
164
  :param fastq_set_id:
151
165
  :return:
@@ -156,6 +170,16 @@ def link_fastq_list_row_to_fastq_set(fastq_id: str, fastq_set_id: str) -> FastqS
156
170
 
157
171
 
158
172
  def unlink_fastq_list_row_from_fastq_set(fastq_id: str, fastq_set_id: str) -> FastqSet:
173
+ """
174
+ Deprecated: Use `unlink_fastq_from_fastq_set` instead.
175
+ """
176
+ return unlink_fastq_from_fastq_set(
177
+ fastq_id=fastq_id,
178
+ fastq_set_id=fastq_set_id
179
+ )
180
+
181
+
182
+ def unlink_fastq_from_fastq_set(fastq_id: str, fastq_set_id: str) -> FastqSet:
159
183
  """
160
184
  Unlink a fastq id from a fastq set.
161
185
 
@@ -8,15 +8,16 @@ Workflow helpers - a collection of helper functions for the workflow
8
8
 
9
9
  # Standard library imports
10
10
  from typing import List
11
+
11
12
  # Local imports
12
13
  from . import get_fastq_request
13
- from .globals import FASTQ_LIST_ROW_ENDPOINT, FASTQ_SET_ENDPOINT
14
+ from .globals import FASTQ_ENDPOINT, FASTQ_SET_ENDPOINT
14
15
  from .models import FastqListRowDict
15
16
 
16
17
 
17
18
  def to_fastq_list_row(fastq_id) -> FastqListRowDict:
18
19
  return get_fastq_request(
19
- f"{FASTQ_LIST_ROW_ENDPOINT}/{fastq_id}/toFastqListRow"
20
+ f"{FASTQ_ENDPOINT}/{fastq_id}/toFastqListRow"
20
21
  )
21
22
 
22
23
  def to_fastq_list_rows(fastq_set_id: str) -> List[FastqListRowDict]:
@@ -6,7 +6,10 @@ from typing import Dict, Optional
6
6
  # Local imports
7
7
  from .globals import FASTQ_UNARCHIVING_SUBDOMAIN_NAME
8
8
  from ..utils.requests_helpers import (
9
- get_request_response_results, get_url, patch_request, get_request
9
+ get_request_response_results,
10
+ get_url,
11
+ patch_request,
12
+ get_request
10
13
  )
11
14
 
12
15
 
@@ -16,12 +16,12 @@ def create_job(fastq_ids: List[str], job_type: Optional[JobType] = None) -> Job:
16
16
  Create the job
17
17
  """
18
18
  if job_type is None:
19
- job_type = JobType.S3_UNARCHIVING
19
+ job_type = 'S3_UNARCHIVING'
20
20
 
21
21
  return fastq_unarchiving_post_request(
22
22
  JOB_ENDPOINT,
23
23
  params={
24
24
  "fastqIds": fastq_ids,
25
- "jobType": job_type.value
25
+ "jobType": job_type
26
26
  }
27
27
  )
@@ -1,7 +1,6 @@
1
1
  #!/usr/bin/env python3
2
2
 
3
3
  import re
4
- from enum import Enum
5
4
 
6
5
  # AWS PARAMETERS
7
6
  FASTQ_UNARCHIVING_SUBDOMAIN_NAME = "fastq-unarchiving"
@@ -12,10 +11,3 @@ JOB_ENDPOINT = "api/v1/jobs"
12
11
  # REGEX
13
12
  ORCABUS_ULID_REGEX_MATCH = re.compile(r'^[a-z0-9]{3}\.[A-Z0-9]{26}$')
14
13
 
15
-
16
- class JobStatus(Enum):
17
- PENDING = "PENDING"
18
- RUNNING = "RUNNING"
19
- FAILED = "FAILED"
20
- ABORTED = "ABORTED"
21
- SUCCEEDED = "SUCCEEDED"
@@ -13,28 +13,27 @@
13
13
 
14
14
  from typing import (
15
15
  TypedDict, NotRequired,
16
+ Literal
16
17
  )
17
18
 
18
- from enum import Enum
19
+ JobType = Literal[
20
+ "S3_UNARCHIVING",
21
+ ]
19
22
 
20
23
 
21
- class JobType(Enum):
22
- S3_UNARCHIVING = "S3_UNARCHIVING"
23
-
24
-
25
- class JobStatus(Enum):
26
- PENDING = "PENDING"
27
- RUNNING = "RUNNING"
28
- FAILED = "FAILED"
29
- ABORTED = "ABORTED"
30
- SUCCEEDED = "SUCCEEDED"
31
-
24
+ JobStatusType = Literal[
25
+ "PENDING",
26
+ "RUNNING",
27
+ "FAILED",
28
+ "ABORTED",
29
+ "SUCCEEDED",
30
+ ]
32
31
 
33
32
  class Job(TypedDict):
34
33
  id: str
35
34
  jobType: JobType
36
35
  stepsExecutionArn: str
37
- status: JobStatus
36
+ status: JobStatusType
38
37
  startTime: str
39
38
  endTime: str
40
39
  errorMessages: str
@@ -42,7 +41,7 @@ class Job(TypedDict):
42
41
 
43
42
  class JobQueryParameters(TypedDict):
44
43
  fastqId: NotRequired[str]
45
- status: NotRequired[JobStatus]
44
+ status: NotRequired[JobStatusType]
46
45
  createdAfter: NotRequired[str]
47
46
  createdBefore: NotRequired[str]
48
47
  completedAfter: NotRequired[str]
@@ -14,7 +14,7 @@ from typing import List, Unpack
14
14
 
15
15
  # Local imports
16
16
  from . import get_fastq_unarchiving_request_response_results
17
- from .models import Job, JobStatus, JobQueryParameters
17
+ from .models import Job, JobStatusType, JobQueryParameters
18
18
 
19
19
  from .globals import JOB_ENDPOINT
20
20
 
@@ -40,7 +40,7 @@ def get_unarchiving_job_list(**kwargs: Unpack[JobQueryParameters]) -> List[Job]:
40
40
 
41
41
  def get_job_list_for_fastq(
42
42
  fastq_id: str,
43
- job_status: JobStatus
43
+ job_status: JobStatusType
44
44
  ) -> List[Job]:
45
45
  """
46
46
  Check if fastq in job list
@@ -48,5 +48,5 @@ def get_job_list_for_fastq(
48
48
  """
49
49
  return get_unarchiving_job_list(
50
50
  fastqId=fastq_id,
51
- status=job_status.value
51
+ status=job_status
52
52
  )
@@ -17,13 +17,13 @@ from typing import Optional
17
17
 
18
18
  # Local imports
19
19
  from . import fastq_unarchiving_patch_request
20
- from .globals import JobStatus, JOB_ENDPOINT
21
- from .models import Job
20
+ from .globals import JOB_ENDPOINT
21
+ from .models import Job, JobStatusType
22
22
 
23
23
 
24
24
  def update_status(
25
25
  job_id: str,
26
- job_status: JobStatus,
26
+ job_status: JobStatusType,
27
27
  error_message: Optional[str] = None
28
28
  ) -> Job:
29
29
  """
@@ -40,10 +40,12 @@ def get_file_manager_request(
40
40
 
41
41
  def file_manager_patch_request(
42
42
  endpoint: str,
43
+ json_data: Optional[Dict] = None,
43
44
  params: Optional[Dict] = None
44
45
  ):
45
46
  return patch_request(
46
47
  get_file_manager_url(endpoint),
48
+ json_data=json_data,
47
49
  params=params
48
50
  )
49
51
 
@@ -1,24 +1,30 @@
1
1
  #!/usr/bin/env python3
2
+
3
+ # Standard imports
4
+ import json
2
5
  from functools import reduce
3
6
  from operator import concat
4
7
  from typing import List, Dict, Union
5
8
  import typing
6
-
7
9
  import boto3
10
+ from datetime import datetime, timedelta, timezone
11
+ from urllib.parse import urlparse
12
+ from itertools import batched
8
13
 
14
+ # Local imports
9
15
  from .errors import S3FileNotFoundError, S3DuplicateFileCopyError
10
- from .models import FileObject
16
+ from .models import FileObject, StorageClassPriority
11
17
  from ..utils.miscell import get_bucket_key_pair_from_uri
12
- from . import get_file_manager_request_response_results, get_file_manager_request, file_manager_patch_request
18
+ from . import (
19
+ get_file_manager_request_response_results,
20
+ get_file_manager_request,
21
+ file_manager_patch_request
22
+ )
13
23
  from .globals import (
14
24
  S3_LIST_ENDPOINT,
15
25
  S3_BUCKETS_BY_ACCOUNT_ID,
16
- S3_PREFIXES_BY_ACCOUNT_ID,
17
- StorageEnum, StoragePriority,
26
+ S3_PREFIXES_BY_ACCOUNT_ID
18
27
  )
19
- from datetime import datetime, timedelta, timezone
20
- from urllib.parse import urlparse
21
- from itertools import batched
22
28
 
23
29
  if typing.TYPE_CHECKING:
24
30
  from mypy_boto3_sts import STSClient
@@ -94,7 +100,7 @@ def get_file_object_from_ingest_id(ingest_id: str, **kwargs) -> FileObject:
94
100
  ))
95
101
 
96
102
  # Order by storage class
97
- file_objects_list.sort(key=lambda file_obj_iter_: StoragePriority[StorageEnum(file_obj_iter_['storageClass']).name].value)
103
+ file_objects_list.sort(key=lambda file_obj_iter_: StorageClassPriority[file_obj_iter_['storageClass']])
98
104
 
99
105
  # Return as a FileObject model
100
106
  return file_objects_list[0]
@@ -263,8 +269,9 @@ def get_s3_objs_from_ingest_ids_map(ingest_ids: List[str], **kwargs) -> List[Dic
263
269
  continue
264
270
 
265
271
  s3_objects_match.sort(
266
- key=lambda s3_object_iter_: StoragePriority[
267
- StorageEnum(s3_object_iter_['fileObject']['storageClass']).name].value
272
+ key=lambda s3_object_iter_: StorageClassPriority[
273
+ s3_object_iter_['fileObject']['storageClass']
274
+ ]
268
275
  )
269
276
 
270
277
  s3_objects_by_ingest_id_filtered.append(
@@ -1,5 +1,4 @@
1
1
  #!/usr/bin/env python
2
- from enum import Enum
3
2
 
4
3
  # AWS PARAMETERS
5
4
  FILEMANAGER_SUBDOMAIN_NAME = "file"
@@ -41,30 +40,3 @@ S3_PREFIXES_BY_ACCOUNT_ID = {
41
40
  },
42
41
  }
43
42
 
44
- # FROM FileManager Schema
45
- # "DeepArchive"
46
- # "Glacier"
47
- # "GlacierIr"
48
- # "IntelligentTiering"
49
- # "OnezoneIa"
50
- # "Outposts"
51
- # "ReducedRedundancy"
52
- # "Snow"
53
- # "Standard"
54
- # "StandardIa"
55
- class StorageEnum(Enum):
56
- STANDARD = "Standard"
57
- STANDARD_IA = "StandardIa"
58
- INTELLIGENT_TIERING = "IntelligentTiering"
59
- GLACIER_INSTANT_RETRIEVAL = "GlacierIr"
60
- GLACIER = "Glacier"
61
- DEEP_ARCHIVE = "DeepArchive"
62
-
63
-
64
- class StoragePriority(Enum):
65
- STANDARD = 1
66
- STANDARD_IA = 2
67
- INTELLIGENT_TIERING = 3
68
- GLACIER_INSTANT_RETRIEVAL = 4
69
- GLACIER = 5
70
- DEEP_ARCHIVE = 6
@@ -1,4 +1,4 @@
1
- from typing import Optional, TypedDict, Dict
1
+ from typing import Optional, TypedDict, Dict, Literal
2
2
 
3
3
  """
4
4
  Example File Object response
@@ -26,6 +26,23 @@ Example File Object response
26
26
  }
27
27
  """
28
28
 
29
+ StorageClassType = Literal[
30
+ "Standard",
31
+ "StandardIa",
32
+ "IntelligentTiering",
33
+ "GlacierIr",
34
+ "Glacier",
35
+ "DeepArchive",
36
+ ]
37
+
38
+ StorageClassPriority: Dict[StorageClassType, int] = {
39
+ "Standard": 1,
40
+ "StandardIa": 2,
41
+ "IntelligentTiering": 3,
42
+ "GlacierIr": 4,
43
+ "Glacier": 5,
44
+ "DeepArchive": 6,
45
+ }
29
46
 
30
47
  class FileObject(TypedDict):
31
48
  # Identifier
@@ -47,7 +64,7 @@ class FileObject(TypedDict):
47
64
  numberReordered: int
48
65
  sequencer: str
49
66
  size: int
50
- storageClass: str
67
+ storageClass: StorageClassType
51
68
 
52
69
  # Attribute attributes
53
70
  attributes: Optional[Dict]
@@ -9,6 +9,8 @@ from copy import deepcopy
9
9
 
10
10
  from requests import HTTPError
11
11
 
12
+ from fastapi.encoders import jsonable_encoder
13
+
12
14
  # Locals
13
15
  from .aws_helpers import (
14
16
  get_orcabus_token, get_hostname
@@ -103,7 +105,11 @@ def get_request(url: str, params: Optional[Dict] = None) -> Dict:
103
105
  return response.json()
104
106
 
105
107
 
106
- def patch_request(url: str, params: Optional[Dict] = None) -> Dict:
108
+ def patch_request(
109
+ url: str,
110
+ json_data: Optional[Dict] = None,
111
+ params: Optional[Dict] = None
112
+ ) -> Dict:
107
113
  # Get authorization header
108
114
  headers = {
109
115
  "Authorization": f"Bearer {get_orcabus_token()}"
@@ -119,7 +125,8 @@ def patch_request(url: str, params: Optional[Dict] = None) -> Dict:
119
125
  response = requests.patch(
120
126
  url,
121
127
  headers=headers,
122
- json=req_params
128
+ params=req_params,
129
+ json=json_data
123
130
  )
124
131
 
125
132
  try:
@@ -13,7 +13,7 @@ const aws_cloudwatch_actions_1 = require("aws-cdk-lib/aws-cloudwatch-actions");
13
13
  * SNS topic.
14
14
  */
15
15
  class MonitoredQueue extends constructs_1.Construct {
16
- static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.monitoredQueue.MonitoredQueue", version: "0.0.34" };
16
+ static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.monitoredQueue.MonitoredQueue", version: "0.0.35" };
17
17
  queue;
18
18
  deadLetterQueue;
19
19
  alarm;
@@ -7,7 +7,7 @@ const aws_iam_1 = require("aws-cdk-lib/aws-iam");
7
7
  * A construct which represents a named role that a Lambda function can assume.
8
8
  */
9
9
  class NamedLambdaRole extends aws_iam_1.Role {
10
- static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.namedLambdaRole.NamedLambdaRole", version: "0.0.34" };
10
+ static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.namedLambdaRole.NamedLambdaRole", version: "0.0.35" };
11
11
  constructor(scope, id, props) {
12
12
  super(scope, id, {
13
13
  assumedBy: new aws_iam_1.ServicePrincipal("lambda.amazonaws.com"),
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@orcabus/platform-cdk-constructs",
3
- "version": "0.0.34",
3
+ "version": "0.0.35",
4
4
  "description": "CDK Package for OrcaBus infrastructure.",
5
5
  "main": "index.ts",
6
6
  "types": "index.d.ts",
@@ -15,7 +15,7 @@ const aws_cdk_lib_1 = require("aws-cdk-lib");
15
15
  * to the provider framework.
16
16
  */
17
17
  class ProviderFunction extends constructs_1.Construct {
18
- static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.provider.ProviderFunction", version: "0.0.34" };
18
+ static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.provider.ProviderFunction", version: "0.0.35" };
19
19
  _function;
20
20
  _response;
21
21
  constructor(scope, id, props) {
@@ -23,7 +23,7 @@ exports.VPC_LOOKUP_PROPS = {
23
23
  * Helper for looking up the shared OrcaBus VPC.
24
24
  */
25
25
  class OrcaBusVpc {
26
- static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.sharedConfig.networking.OrcaBusVpc", version: "0.0.34" };
26
+ static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.sharedConfig.networking.OrcaBusVpc", version: "0.0.35" };
27
27
  /**
28
28
  * The shared VPC that is used by OrcaBus.
29
29
  * @param scope
@@ -42,7 +42,7 @@ exports.SHARED_SECURITY_GROUP_NAME = "OrcaBusSharedComputeSecurityGroup";
42
42
  * Helper for looking up the shared compute security group by name.
43
43
  */
44
44
  class OrcaBusSharedComputeSecurityGroup {
45
- static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.sharedConfig.networking.OrcaBusSharedComputeSecurityGroup", version: "0.0.34" };
45
+ static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.sharedConfig.networking.OrcaBusSharedComputeSecurityGroup", version: "0.0.35" };
46
46
  /**
47
47
  * The shared security group that is used by compute resources to access the database.
48
48
  * @param scope
@@ -11,7 +11,7 @@ exports.SLACK_ALERTS_SNS_TOPIC = "AwsChatBotTopic-alerts";
11
11
  * A helper class to construct the arn for the slack SNS topic.
12
12
  */
13
13
  class SlackAlerts {
14
- static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.sharedConfig.slack.SlackAlerts", version: "0.0.34" };
14
+ static [JSII_RTTI_SYMBOL_1] = { fqn: "@orcabus/platform-cdk-constructs.sharedConfig.slack.SlackAlerts", version: "0.0.35" };
15
15
  /**
16
16
  * Format the ARN for the slack alerts SNS topic for the current stack.
17
17
  * @param stack