@orcabus/platform-cdk-constructs 0.0.6 → 0.0.7-alpha.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/.jsii +858 -6
  2. package/api-gateway/api-gateway.js +22 -4
  3. package/deployment-stack-pipeline/pipeline.js +6 -4
  4. package/dynamodb/config.d.ts +3 -0
  5. package/dynamodb/config.js +7 -0
  6. package/dynamodb/index.d.ts +27 -0
  7. package/dynamodb/index.js +106 -0
  8. package/ecs/config.d.ts +4 -0
  9. package/ecs/config.js +8 -0
  10. package/ecs/index.d.ts +46 -0
  11. package/ecs/index.js +103 -0
  12. package/index.d.ts +5 -2
  13. package/index.js +47 -5
  14. package/index.ts +16 -2
  15. package/lambda/build_python/Dockerfile +60 -0
  16. package/lambda/config.d.ts +11 -0
  17. package/lambda/config.js +19 -0
  18. package/lambda/index.d.ts +59 -0
  19. package/lambda/index.js +242 -0
  20. package/lambda/layers/mart_tools/poetry.lock +303 -0
  21. package/lambda/layers/mart_tools/pyproject.toml +27 -0
  22. package/lambda/layers/mart_tools/src/mart_tools/__init__.py +0 -0
  23. package/lambda/layers/mart_tools/src/mart_tools/mart/__init__.py +8 -0
  24. package/lambda/layers/mart_tools/src/mart_tools/mart/aws_helpers.py +79 -0
  25. package/lambda/layers/mart_tools/src/mart_tools/mart/dataframe_helpers.py +29 -0
  26. package/lambda/layers/mart_tools/src/mart_tools/mart/globals.py +5 -0
  27. package/lambda/layers/mart_tools/src/mart_tools/mart/models.py +71 -0
  28. package/lambda/layers/orcabus_api_tools/poetry.lock +273 -0
  29. package/lambda/layers/orcabus_api_tools/pyproject.toml +27 -0
  30. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/__init__.py +0 -0
  31. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/__init__.py +172 -0
  32. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/create_helpers.py +47 -0
  33. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/globals.py +13 -0
  34. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/job_helpers.py +53 -0
  35. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/models.py +253 -0
  36. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/query_helpers.py +248 -0
  37. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/update_helpers.py +221 -0
  38. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq/workflow_helpers.py +25 -0
  39. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq_unarchiving/__init__.py +92 -0
  40. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq_unarchiving/create_helpers.py +27 -0
  41. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq_unarchiving/globals.py +21 -0
  42. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq_unarchiving/models.py +51 -0
  43. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq_unarchiving/query_helpers.py +52 -0
  44. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/fastq_unarchiving/update_helpers.py +45 -0
  45. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/filemanager/__init__.py +98 -0
  46. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/filemanager/errors.py +45 -0
  47. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/filemanager/file_helpers.py +341 -0
  48. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/filemanager/globals.py +70 -0
  49. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/filemanager/models.py +59 -0
  50. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/mart/__init__.py +8 -0
  51. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/mart/aws_helpers.py +79 -0
  52. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/mart/dataframe_helpers.py +29 -0
  53. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/mart/globals.py +5 -0
  54. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/mart/models.py +71 -0
  55. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/metadata/__init__.py +250 -0
  56. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/metadata/contact_helpers.py +109 -0
  57. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/metadata/errors.py +104 -0
  58. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/metadata/globals.py +16 -0
  59. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/metadata/individual_helpers.py +139 -0
  60. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/metadata/library_helpers.py +196 -0
  61. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/metadata/lims_helpers.py +36 -0
  62. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/metadata/models.py +112 -0
  63. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/metadata/project_helpers.py +129 -0
  64. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/metadata/sample_helpers.py +132 -0
  65. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/metadata/subject_helpers.py +151 -0
  66. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/sequence/__init__.py +15 -0
  67. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/sequence/globals.py +2 -0
  68. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/sequence/models.py +44 -0
  69. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/sequence/sequence_helpers.py +62 -0
  70. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/utils/__init__.py +0 -0
  71. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/utils/aws_helpers.py +123 -0
  72. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/utils/miscell.py +17 -0
  73. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/utils/requests_helpers.py +163 -0
  74. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/workflow/__init__.py +0 -0
  75. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/workflow/errors.py +37 -0
  76. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/workflow/metadata_helpers.py +28 -0
  77. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/workflow/models.py +85 -0
  78. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/workflow/payload_helpers.py +64 -0
  79. package/lambda/layers/orcabus_api_tools/src/orcabus_api_tools/workflow/workflow_run_helpers.py +80 -0
  80. package/package.json +13 -7
  81. package/typedoc.json +3 -0
  82. package/utils/index.d.ts +3 -0
  83. package/utils/index.js +50 -1
@@ -0,0 +1,303 @@
1
+ # This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand.
2
+
3
+ [[package]]
4
+ name = "colorama"
5
+ version = "0.4.6"
6
+ description = "Cross-platform colored terminal text."
7
+ optional = false
8
+ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
9
+ groups = ["dev"]
10
+ markers = "sys_platform == \"win32\""
11
+ files = [
12
+ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"},
13
+ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
14
+ ]
15
+
16
+ [[package]]
17
+ name = "iniconfig"
18
+ version = "2.1.0"
19
+ description = "brain-dead simple config-ini parsing"
20
+ optional = false
21
+ python-versions = ">=3.8"
22
+ groups = ["dev"]
23
+ files = [
24
+ {file = "iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760"},
25
+ {file = "iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7"},
26
+ ]
27
+
28
+ [[package]]
29
+ name = "mypy-boto3-secretsmanager"
30
+ version = "1.38.0"
31
+ description = "Type annotations for boto3 SecretsManager 1.38.0 service generated with mypy-boto3-builder 8.10.1"
32
+ optional = false
33
+ python-versions = ">=3.8"
34
+ groups = ["dev"]
35
+ files = [
36
+ {file = "mypy_boto3_secretsmanager-1.38.0-py3-none-any.whl", hash = "sha256:48d5057450ee307b132ce2d0976233a2c5331616fabdf423ecbc103f7431dd5e"},
37
+ {file = "mypy_boto3_secretsmanager-1.38.0.tar.gz", hash = "sha256:1666108e70f03e4dc1de449388d7facb77aba231a026bac0c3240fc27fd31a98"},
38
+ ]
39
+
40
+ [[package]]
41
+ name = "mypy-boto3-ssm"
42
+ version = "1.38.5"
43
+ description = "Type annotations for boto3 SSM 1.38.5 service generated with mypy-boto3-builder 8.10.1"
44
+ optional = false
45
+ python-versions = ">=3.8"
46
+ groups = ["dev"]
47
+ files = [
48
+ {file = "mypy_boto3_ssm-1.38.5-py3-none-any.whl", hash = "sha256:1bb0f932bee9038a53ab02781f959fc553a5d7f5e9d7cba56f998d0eb0a5878f"},
49
+ {file = "mypy_boto3_ssm-1.38.5.tar.gz", hash = "sha256:e95bbad7d2f6b4849bc946eb9bbcc1f134cbdaafb172c365bedecdb3104eee0e"},
50
+ ]
51
+
52
+ [[package]]
53
+ name = "numpy"
54
+ version = "2.2.5"
55
+ description = "Fundamental package for array computing in Python"
56
+ optional = false
57
+ python-versions = ">=3.10"
58
+ groups = ["main"]
59
+ files = [
60
+ {file = "numpy-2.2.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1f4a922da1729f4c40932b2af4fe84909c7a6e167e6e99f71838ce3a29f3fe26"},
61
+ {file = "numpy-2.2.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b6f91524d31b34f4a5fee24f5bc16dcd1491b668798b6d85585d836c1e633a6a"},
62
+ {file = "numpy-2.2.5-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:19f4718c9012e3baea91a7dba661dcab2451cda2550678dc30d53acb91a7290f"},
63
+ {file = "numpy-2.2.5-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:eb7fd5b184e5d277afa9ec0ad5e4eb562ecff541e7f60e69ee69c8d59e9aeaba"},
64
+ {file = "numpy-2.2.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6413d48a9be53e183eb06495d8e3b006ef8f87c324af68241bbe7a39e8ff54c3"},
65
+ {file = "numpy-2.2.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7451f92eddf8503c9b8aa4fe6aa7e87fd51a29c2cfc5f7dbd72efde6c65acf57"},
66
+ {file = "numpy-2.2.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0bcb1d057b7571334139129b7f941588f69ce7c4ed15a9d6162b2ea54ded700c"},
67
+ {file = "numpy-2.2.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:36ab5b23915887543441efd0417e6a3baa08634308894316f446027611b53bf1"},
68
+ {file = "numpy-2.2.5-cp310-cp310-win32.whl", hash = "sha256:422cc684f17bc963da5f59a31530b3936f57c95a29743056ef7a7903a5dbdf88"},
69
+ {file = "numpy-2.2.5-cp310-cp310-win_amd64.whl", hash = "sha256:e4f0b035d9d0ed519c813ee23e0a733db81ec37d2e9503afbb6e54ccfdee0fa7"},
70
+ {file = "numpy-2.2.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c42365005c7a6c42436a54d28c43fe0e01ca11eb2ac3cefe796c25a5f98e5e9b"},
71
+ {file = "numpy-2.2.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:498815b96f67dc347e03b719ef49c772589fb74b8ee9ea2c37feae915ad6ebda"},
72
+ {file = "numpy-2.2.5-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:6411f744f7f20081b1b4e7112e0f4c9c5b08f94b9f086e6f0adf3645f85d3a4d"},
73
+ {file = "numpy-2.2.5-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:9de6832228f617c9ef45d948ec1cd8949c482238d68b2477e6f642c33a7b0a54"},
74
+ {file = "numpy-2.2.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:369e0d4647c17c9363244f3468f2227d557a74b6781cb62ce57cf3ef5cc7c610"},
75
+ {file = "numpy-2.2.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:262d23f383170f99cd9191a7c85b9a50970fe9069b2f8ab5d786eca8a675d60b"},
76
+ {file = "numpy-2.2.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:aa70fdbdc3b169d69e8c59e65c07a1c9351ceb438e627f0fdcd471015cd956be"},
77
+ {file = "numpy-2.2.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37e32e985f03c06206582a7323ef926b4e78bdaa6915095ef08070471865b906"},
78
+ {file = "numpy-2.2.5-cp311-cp311-win32.whl", hash = "sha256:f5045039100ed58fa817a6227a356240ea1b9a1bc141018864c306c1a16d4175"},
79
+ {file = "numpy-2.2.5-cp311-cp311-win_amd64.whl", hash = "sha256:b13f04968b46ad705f7c8a80122a42ae8f620536ea38cf4bdd374302926424dd"},
80
+ {file = "numpy-2.2.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ee461a4eaab4f165b68780a6a1af95fb23a29932be7569b9fab666c407969051"},
81
+ {file = "numpy-2.2.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ec31367fd6a255dc8de4772bd1658c3e926d8e860a0b6e922b615e532d320ddc"},
82
+ {file = "numpy-2.2.5-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:47834cde750d3c9f4e52c6ca28a7361859fcaf52695c7dc3cc1a720b8922683e"},
83
+ {file = "numpy-2.2.5-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2c1a1c6ccce4022383583a6ded7bbcda22fc635eb4eb1e0a053336425ed36dfa"},
84
+ {file = "numpy-2.2.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9d75f338f5f79ee23548b03d801d28a505198297534f62416391857ea0479571"},
85
+ {file = "numpy-2.2.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a801fef99668f309b88640e28d261991bfad9617c27beda4a3aec4f217ea073"},
86
+ {file = "numpy-2.2.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:abe38cd8381245a7f49967a6010e77dbf3680bd3627c0fe4362dd693b404c7f8"},
87
+ {file = "numpy-2.2.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5a0ac90e46fdb5649ab6369d1ab6104bfe5854ab19b645bf5cda0127a13034ae"},
88
+ {file = "numpy-2.2.5-cp312-cp312-win32.whl", hash = "sha256:0cd48122a6b7eab8f06404805b1bd5856200e3ed6f8a1b9a194f9d9054631beb"},
89
+ {file = "numpy-2.2.5-cp312-cp312-win_amd64.whl", hash = "sha256:ced69262a8278547e63409b2653b372bf4baff0870c57efa76c5703fd6543282"},
90
+ {file = "numpy-2.2.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:059b51b658f4414fff78c6d7b1b4e18283ab5fa56d270ff212d5ba0c561846f4"},
91
+ {file = "numpy-2.2.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:47f9ed103af0bc63182609044b0490747e03bd20a67e391192dde119bf43d52f"},
92
+ {file = "numpy-2.2.5-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:261a1ef047751bb02f29dfe337230b5882b54521ca121fc7f62668133cb119c9"},
93
+ {file = "numpy-2.2.5-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4520caa3807c1ceb005d125a75e715567806fed67e315cea619d5ec6e75a4191"},
94
+ {file = "numpy-2.2.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d14b17b9be5f9c9301f43d2e2a4886a33b53f4e6fdf9ca2f4cc60aeeee76372"},
95
+ {file = "numpy-2.2.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ba321813a00e508d5421104464510cc962a6f791aa2fca1c97b1e65027da80d"},
96
+ {file = "numpy-2.2.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4cbdef3ddf777423060c6f81b5694bad2dc9675f110c4b2a60dc0181543fac7"},
97
+ {file = "numpy-2.2.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:54088a5a147ab71a8e7fdfd8c3601972751ded0739c6b696ad9cb0343e21ab73"},
98
+ {file = "numpy-2.2.5-cp313-cp313-win32.whl", hash = "sha256:c8b82a55ef86a2d8e81b63da85e55f5537d2157165be1cb2ce7cfa57b6aef38b"},
99
+ {file = "numpy-2.2.5-cp313-cp313-win_amd64.whl", hash = "sha256:d8882a829fd779f0f43998e931c466802a77ca1ee0fe25a3abe50278616b1471"},
100
+ {file = "numpy-2.2.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:e8b025c351b9f0e8b5436cf28a07fa4ac0204d67b38f01433ac7f9b870fa38c6"},
101
+ {file = "numpy-2.2.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:8dfa94b6a4374e7851bbb6f35e6ded2120b752b063e6acdd3157e4d2bb922eba"},
102
+ {file = "numpy-2.2.5-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:97c8425d4e26437e65e1d189d22dff4a079b747ff9c2788057bfb8114ce1e133"},
103
+ {file = "numpy-2.2.5-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:352d330048c055ea6db701130abc48a21bec690a8d38f8284e00fab256dc1376"},
104
+ {file = "numpy-2.2.5-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b4c0773b6ada798f51f0f8e30c054d32304ccc6e9c5d93d46cb26f3d385ab19"},
105
+ {file = "numpy-2.2.5-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:55f09e00d4dccd76b179c0f18a44f041e5332fd0e022886ba1c0bbf3ea4a18d0"},
106
+ {file = "numpy-2.2.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:02f226baeefa68f7d579e213d0f3493496397d8f1cff5e2b222af274c86a552a"},
107
+ {file = "numpy-2.2.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c26843fd58f65da9491165072da2cccc372530681de481ef670dcc8e27cfb066"},
108
+ {file = "numpy-2.2.5-cp313-cp313t-win32.whl", hash = "sha256:1a161c2c79ab30fe4501d5a2bbfe8b162490757cf90b7f05be8b80bc02f7bb8e"},
109
+ {file = "numpy-2.2.5-cp313-cp313t-win_amd64.whl", hash = "sha256:d403c84991b5ad291d3809bace5e85f4bbf44a04bdc9a88ed2bb1807b3360bb8"},
110
+ {file = "numpy-2.2.5-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b4ea7e1cff6784e58fe281ce7e7f05036b3e1c89c6f922a6bfbc0a7e8768adbe"},
111
+ {file = "numpy-2.2.5-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:d7543263084a85fbc09c704b515395398d31d6395518446237eac219eab9e55e"},
112
+ {file = "numpy-2.2.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0255732338c4fdd00996c0421884ea8a3651eea555c3a56b84892b66f696eb70"},
113
+ {file = "numpy-2.2.5-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d2e3bdadaba0e040d1e7ab39db73e0afe2c74ae277f5614dad53eadbecbbb169"},
114
+ {file = "numpy-2.2.5.tar.gz", hash = "sha256:a9c0d994680cd991b1cb772e8b297340085466a6fe964bc9d4e80f5e2f43c291"},
115
+ ]
116
+
117
+ [[package]]
118
+ name = "packaging"
119
+ version = "25.0"
120
+ description = "Core utilities for Python packages"
121
+ optional = false
122
+ python-versions = ">=3.8"
123
+ groups = ["dev"]
124
+ files = [
125
+ {file = "packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484"},
126
+ {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"},
127
+ ]
128
+
129
+ [[package]]
130
+ name = "pandas"
131
+ version = "2.2.3"
132
+ description = "Powerful data structures for data analysis, time series, and statistics"
133
+ optional = false
134
+ python-versions = ">=3.9"
135
+ groups = ["main"]
136
+ files = [
137
+ {file = "pandas-2.2.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1948ddde24197a0f7add2bdc4ca83bf2b1ef84a1bc8ccffd95eda17fd836ecb5"},
138
+ {file = "pandas-2.2.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:381175499d3802cde0eabbaf6324cce0c4f5d52ca6f8c377c29ad442f50f6348"},
139
+ {file = "pandas-2.2.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d9c45366def9a3dd85a6454c0e7908f2b3b8e9c138f5dc38fed7ce720d8453ed"},
140
+ {file = "pandas-2.2.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86976a1c5b25ae3f8ccae3a5306e443569ee3c3faf444dfd0f41cda24667ad57"},
141
+ {file = "pandas-2.2.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b8661b0238a69d7aafe156b7fa86c44b881387509653fdf857bebc5e4008ad42"},
142
+ {file = "pandas-2.2.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:37e0aced3e8f539eccf2e099f65cdb9c8aa85109b0be6e93e2baff94264bdc6f"},
143
+ {file = "pandas-2.2.3-cp310-cp310-win_amd64.whl", hash = "sha256:56534ce0746a58afaf7942ba4863e0ef81c9c50d3f0ae93e9497d6a41a057645"},
144
+ {file = "pandas-2.2.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:66108071e1b935240e74525006034333f98bcdb87ea116de573a6a0dccb6c039"},
145
+ {file = "pandas-2.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7c2875855b0ff77b2a64a0365e24455d9990730d6431b9e0ee18ad8acee13dbd"},
146
+ {file = "pandas-2.2.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd8d0c3be0515c12fed0bdbae072551c8b54b7192c7b1fda0ba56059a0179698"},
147
+ {file = "pandas-2.2.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c124333816c3a9b03fbeef3a9f230ba9a737e9e5bb4060aa2107a86cc0a497fc"},
148
+ {file = "pandas-2.2.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:63cc132e40a2e084cf01adf0775b15ac515ba905d7dcca47e9a251819c575ef3"},
149
+ {file = "pandas-2.2.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:29401dbfa9ad77319367d36940cd8a0b3a11aba16063e39632d98b0e931ddf32"},
150
+ {file = "pandas-2.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:3fc6873a41186404dad67245896a6e440baacc92f5b716ccd1bc9ed2995ab2c5"},
151
+ {file = "pandas-2.2.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b1d432e8d08679a40e2a6d8b2f9770a5c21793a6f9f47fdd52c5ce1948a5a8a9"},
152
+ {file = "pandas-2.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a5a1595fe639f5988ba6a8e5bc9649af3baf26df3998a0abe56c02609392e0a4"},
153
+ {file = "pandas-2.2.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5de54125a92bb4d1c051c0659e6fcb75256bf799a732a87184e5ea503965bce3"},
154
+ {file = "pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319"},
155
+ {file = "pandas-2.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dfcb5ee8d4d50c06a51c2fffa6cff6272098ad6540aed1a76d15fb9318194d8"},
156
+ {file = "pandas-2.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:062309c1b9ea12a50e8ce661145c6aab431b1e99530d3cd60640e255778bd43a"},
157
+ {file = "pandas-2.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:59ef3764d0fe818125a5097d2ae867ca3fa64df032331b7e0917cf5d7bf66b13"},
158
+ {file = "pandas-2.2.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f00d1345d84d8c86a63e476bb4955e46458b304b9575dcf71102b5c705320015"},
159
+ {file = "pandas-2.2.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3508d914817e153ad359d7e069d752cdd736a247c322d932eb89e6bc84217f28"},
160
+ {file = "pandas-2.2.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22a9d949bfc9a502d320aa04e5d02feab689d61da4e7764b62c30b991c42c5f0"},
161
+ {file = "pandas-2.2.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3a255b2c19987fbbe62a9dfd6cff7ff2aa9ccab3fc75218fd4b7530f01efa24"},
162
+ {file = "pandas-2.2.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:800250ecdadb6d9c78eae4990da62743b857b470883fa27f652db8bdde7f6659"},
163
+ {file = "pandas-2.2.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6374c452ff3ec675a8f46fd9ab25c4ad0ba590b71cf0656f8b6daa5202bca3fb"},
164
+ {file = "pandas-2.2.3-cp313-cp313-win_amd64.whl", hash = "sha256:61c5ad4043f791b61dd4752191d9f07f0ae412515d59ba8f005832a532f8736d"},
165
+ {file = "pandas-2.2.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3b71f27954685ee685317063bf13c7709a7ba74fc996b84fc6821c59b0f06468"},
166
+ {file = "pandas-2.2.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:38cf8125c40dae9d5acc10fa66af8ea6fdf760b2714ee482ca691fc66e6fcb18"},
167
+ {file = "pandas-2.2.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ba96630bc17c875161df3818780af30e43be9b166ce51c9a18c1feae342906c2"},
168
+ {file = "pandas-2.2.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1db71525a1538b30142094edb9adc10be3f3e176748cd7acc2240c2f2e5aa3a4"},
169
+ {file = "pandas-2.2.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:15c0e1e02e93116177d29ff83e8b1619c93ddc9c49083f237d4312337a61165d"},
170
+ {file = "pandas-2.2.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ad5b65698ab28ed8d7f18790a0dc58005c7629f227be9ecc1072aa74c0c1d43a"},
171
+ {file = "pandas-2.2.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc6b93f9b966093cb0fd62ff1a7e4c09e6d546ad7c1de191767baffc57628f39"},
172
+ {file = "pandas-2.2.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5dbca4c1acd72e8eeef4753eeca07de9b1db4f398669d5994086f788a5d7cc30"},
173
+ {file = "pandas-2.2.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8cd6d7cc958a3910f934ea8dbdf17b2364827bb4dafc38ce6eef6bb3d65ff09c"},
174
+ {file = "pandas-2.2.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99df71520d25fade9db7c1076ac94eb994f4d2673ef2aa2e86ee039b6746d20c"},
175
+ {file = "pandas-2.2.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:31d0ced62d4ea3e231a9f228366919a5ea0b07440d9d4dac345376fd8e1477ea"},
176
+ {file = "pandas-2.2.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7eee9e7cea6adf3e3d24e304ac6b8300646e2a5d1cd3a3c2abed9101b0846761"},
177
+ {file = "pandas-2.2.3-cp39-cp39-win_amd64.whl", hash = "sha256:4850ba03528b6dd51d6c5d273c46f183f39a9baf3f0143e566b89450965b105e"},
178
+ {file = "pandas-2.2.3.tar.gz", hash = "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667"},
179
+ ]
180
+
181
+ [package.dependencies]
182
+ numpy = {version = ">=1.26.0", markers = "python_version >= \"3.12\""}
183
+ python-dateutil = ">=2.8.2"
184
+ pytz = ">=2020.1"
185
+ tzdata = ">=2022.7"
186
+
187
+ [package.extras]
188
+ all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"]
189
+ aws = ["s3fs (>=2022.11.0)"]
190
+ clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"]
191
+ compression = ["zstandard (>=0.19.0)"]
192
+ computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"]
193
+ consortium-standard = ["dataframe-api-compat (>=0.1.7)"]
194
+ excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"]
195
+ feather = ["pyarrow (>=10.0.1)"]
196
+ fss = ["fsspec (>=2022.11.0)"]
197
+ gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"]
198
+ hdf5 = ["tables (>=3.8.0)"]
199
+ html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"]
200
+ mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"]
201
+ output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"]
202
+ parquet = ["pyarrow (>=10.0.1)"]
203
+ performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"]
204
+ plot = ["matplotlib (>=3.6.3)"]
205
+ postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"]
206
+ pyarrow = ["pyarrow (>=10.0.1)"]
207
+ spss = ["pyreadstat (>=1.2.0)"]
208
+ sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"]
209
+ test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"]
210
+ xml = ["lxml (>=4.9.2)"]
211
+
212
+ [[package]]
213
+ name = "pluggy"
214
+ version = "1.5.0"
215
+ description = "plugin and hook calling mechanisms for python"
216
+ optional = false
217
+ python-versions = ">=3.8"
218
+ groups = ["dev"]
219
+ files = [
220
+ {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"},
221
+ {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"},
222
+ ]
223
+
224
+ [package.extras]
225
+ dev = ["pre-commit", "tox"]
226
+ testing = ["pytest", "pytest-benchmark"]
227
+
228
+ [[package]]
229
+ name = "pytest"
230
+ version = "7.4.4"
231
+ description = "pytest: simple powerful testing with Python"
232
+ optional = false
233
+ python-versions = ">=3.7"
234
+ groups = ["dev"]
235
+ files = [
236
+ {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"},
237
+ {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"},
238
+ ]
239
+
240
+ [package.dependencies]
241
+ colorama = {version = "*", markers = "sys_platform == \"win32\""}
242
+ iniconfig = "*"
243
+ packaging = "*"
244
+ pluggy = ">=0.12,<2.0"
245
+
246
+ [package.extras]
247
+ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"]
248
+
249
+ [[package]]
250
+ name = "python-dateutil"
251
+ version = "2.9.0.post0"
252
+ description = "Extensions to the standard Python datetime module"
253
+ optional = false
254
+ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
255
+ groups = ["main"]
256
+ files = [
257
+ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"},
258
+ {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"},
259
+ ]
260
+
261
+ [package.dependencies]
262
+ six = ">=1.5"
263
+
264
+ [[package]]
265
+ name = "pytz"
266
+ version = "2025.2"
267
+ description = "World timezone definitions, modern and historical"
268
+ optional = false
269
+ python-versions = "*"
270
+ groups = ["main"]
271
+ files = [
272
+ {file = "pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00"},
273
+ {file = "pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3"},
274
+ ]
275
+
276
+ [[package]]
277
+ name = "six"
278
+ version = "1.17.0"
279
+ description = "Python 2 and 3 compatibility utilities"
280
+ optional = false
281
+ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
282
+ groups = ["main"]
283
+ files = [
284
+ {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"},
285
+ {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"},
286
+ ]
287
+
288
+ [[package]]
289
+ name = "tzdata"
290
+ version = "2025.2"
291
+ description = "Provider of IANA time zone data"
292
+ optional = false
293
+ python-versions = ">=2"
294
+ groups = ["main"]
295
+ files = [
296
+ {file = "tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8"},
297
+ {file = "tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9"},
298
+ ]
299
+
300
+ [metadata]
301
+ lock-version = "2.1"
302
+ python-versions = "^3.12, <3.13"
303
+ content-hash = "f95ed101ba379dbb255baf87d17800e50a2e9cb04b2d8f264640fb2ba7bce4d2"
@@ -0,0 +1,27 @@
1
+ [build-system]
2
+ requires = ["poetry-core>=1.0.0"]
3
+ build-backend = "poetry.core.masonry.api"
4
+
5
+ [tool.poetry]
6
+ name = "mart_tools"
7
+ version = "1.0.0"
8
+ description = "A collection of secondary-level functions to use to interact with the OrcaBus"
9
+ license = "GPL-3.0-or-later"
10
+ authors = [
11
+ "Alexis Lucattini"
12
+ ]
13
+ homepage = "https://github.com/orcabus/platform-cdk-constructs"
14
+ repository = "https://github.com/orcabus/platform-cdk-constructs"
15
+
16
+ [tool.poetry.dependencies]
17
+ python = "^3.12, <3.13"
18
+ pandas = "^2.2.3"
19
+
20
+ [tool.poetry.group.dev]
21
+ optional = true
22
+
23
+ [tool.poetry.group.dev.dependencies]
24
+ pytest = "^7.0.0" # For testing only
25
+ # For typehinting only, not required at runtime
26
+ mypy-boto3-ssm = "^1.34"
27
+ mypy-boto3-secretsmanager = "^1.34"
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env python3
2
+
3
+ # Set __all__
4
+ from .aws_helpers import run_athena_sql_query
5
+
6
+ __all__ = [
7
+ "run_athena_sql_query"
8
+ ]
@@ -0,0 +1,79 @@
1
+
2
+ # Standard imports
3
+ import typing
4
+ from io import BytesIO
5
+ from os import environ
6
+ from time import sleep
7
+ from typing import Tuple
8
+ from urllib.parse import urlparse
9
+ import boto3
10
+ import pandas as pd
11
+
12
+ # Local imports
13
+ from .globals import DATABASE_ENV_VAR, DATA_SOURCE_ENV_VAR, WORKGROUP_ENV_VAR
14
+
15
+ if typing.TYPE_CHECKING:
16
+ from mypy_boto3_athena import AthenaClient
17
+ from mypy_boto3_s3 import S3Client
18
+
19
+
20
+ def get_athena_client() -> 'AthenaClient':
21
+ return boto3.client('athena')
22
+
23
+
24
+ def get_s3_client() -> 'S3Client':
25
+ return boto3.client('s3')
26
+
27
+
28
+ def get_bucket_key_tuple_from_s3_uri(s3_uri: str) -> Tuple[str, str]:
29
+ urlobj = urlparse(s3_uri)
30
+
31
+ return urlobj.netloc, urlobj.path.lstrip('/')
32
+
33
+
34
+ def run_athena_sql_query(sql_query: str) -> pd.DataFrame:
35
+ from .dataframe_helpers import get_pandas_dtypes_from_athena_query_execution
36
+
37
+ athena_query_execution_id = get_athena_client().start_query_execution(
38
+ QueryString=sql_query,
39
+ QueryExecutionContext={
40
+ "Database": environ[DATABASE_ENV_VAR],
41
+ "Catalog": environ[DATA_SOURCE_ENV_VAR]
42
+ },
43
+ WorkGroup=environ[WORKGROUP_ENV_VAR],
44
+ )['QueryExecutionId']
45
+
46
+ while True:
47
+ status = get_athena_client().get_query_execution(
48
+ QueryExecutionId=athena_query_execution_id
49
+ )['QueryExecution']['Status']['State']
50
+
51
+ if status in ['SUCCEEDED', 'FAILED', 'CANCELLED']:
52
+ break
53
+
54
+ sleep(5)
55
+
56
+ if status in ['FAILED', 'CANCELLED']:
57
+ raise RuntimeError(f"Query failed: {status}")
58
+
59
+ # Get the result info
60
+ column_dtypes = get_pandas_dtypes_from_athena_query_execution(
61
+ query_execution_id=athena_query_execution_id
62
+ )
63
+
64
+ # Get the results
65
+ result_location = get_athena_client().get_query_execution(
66
+ QueryExecutionId=athena_query_execution_id
67
+ )['QueryExecution']['ResultConfiguration']['OutputLocation']
68
+
69
+ bucket, key = get_bucket_key_tuple_from_s3_uri(result_location)
70
+
71
+ return pd.read_csv(
72
+ BytesIO(
73
+ get_s3_client().get_object(
74
+ Bucket=bucket,
75
+ Key=key
76
+ )['Body'].read(),
77
+ ),
78
+ dtype=column_dtypes
79
+ )
@@ -0,0 +1,29 @@
1
+ #!/usr/bin/env python3
2
+
3
+ """
4
+ Collect the mart dataframe but ensuring the correct dtype mappings
5
+
6
+ We do this by
7
+
8
+ 1. Collecting the ColumnInfo from the query results.
9
+ 2. Mapping the 'label' to the 'type'
10
+ 3. Then reading the csv from s3, using the dtype mapping coercions
11
+
12
+ """
13
+
14
+ from .models import ATHENA_TO_DTYPES_MAPPING
15
+
16
+ def get_pandas_dtypes_from_athena_query_execution(query_execution_id):
17
+ from .aws_helpers import get_athena_client
18
+
19
+ column_info_dict = get_athena_client().get_query_results(
20
+ QueryExecutionId=query_execution_id
21
+ )['ResultSet']['ResultSetMetadata']['ColumnInfo']
22
+
23
+ # Create a dictionary mapping the column name to its type
24
+ return dict(map(
25
+ lambda x: (x['Name'], ATHENA_TO_DTYPES_MAPPING.get(x['Type'])),
26
+ column_info_dict
27
+ ))
28
+
29
+
@@ -0,0 +1,5 @@
1
+ # Globals
2
+ # ATHENA
3
+ WORKGROUP_ENV_VAR = 'ATHENA_WORKGROUP_NAME'
4
+ DATA_SOURCE_ENV_VAR = 'ATHENA_DATASOURCE_NAME'
5
+ DATABASE_ENV_VAR = 'ATHENA_DATABASE_NAME'
@@ -0,0 +1,71 @@
1
+ #!/usr/bin/env python
2
+
3
+ from enum import Enum
4
+ from typing import Literal
5
+
6
+
7
+ class AthenaDMLTypes(Enum):
8
+ # https://docs.aws.amazon.com/athena/latest/ug/data-types.html
9
+ BOOLEAN = 'boolean'
10
+ TINYINT = 'tinyint'
11
+ SMALLINT = 'smallint'
12
+ INT = 'int'
13
+ INTEGER = 'integer'
14
+ BIGINT = 'bigint'
15
+ REAL = 'real'
16
+ DOUBLE = 'double'
17
+ DECIMAL = 'decimal'
18
+ CHAR = 'char'
19
+ VARCHAR = 'varchar'
20
+ VARBINARY = 'varbinary'
21
+ TIME = 'time'
22
+ TIME_WITH_TIME_ZONE = 'time with time zone'
23
+ TIMESTAMP = 'timestamp'
24
+ TIMESTAMP_WITH_TIME_ZONE = 'timestamp with time zone'
25
+ ARRAY = 'array'
26
+ MAP = 'map'
27
+ ROW = 'row'
28
+ JSON = 'json'
29
+ UUID = 'uuid'
30
+ IPADDRESS = 'ipaddress'
31
+
32
+
33
+ class PandasDtypes(Enum):
34
+ # https://numpy.org/doc/stable/reference/arrays.dtypes.html
35
+
36
+ # https://pandas.pydata.org/pandas-docs/stable/user_guide/basics.html#dtypes
37
+ DATETIME_NO_TZ = 'datetime64[ns]'
38
+ DATETIME_TZ_AWARE = 'datetime64[ns, UTC]'
39
+ CATEGORICAL = 'category'
40
+ PERIOD = 'period[<freq>]'
41
+ SPARSE = 'Sparse'
42
+ INTERVAL = 'interval'
43
+ NULLABLE_INTEGER = 'Int64'
44
+ NULLABLE_FLOAT = 'Float64'
45
+ STRINGS = 'string'
46
+ BOOLEAN = 'boolean'
47
+ OBJECT = 'object'
48
+
49
+
50
+ ATHENA_TO_DTYPES_MAPPING = {
51
+ AthenaDMLTypes.BOOLEAN.value: PandasDtypes.BOOLEAN.value,
52
+ AthenaDMLTypes.TINYINT.value: PandasDtypes.NULLABLE_INTEGER.value,
53
+ AthenaDMLTypes.SMALLINT.value: PandasDtypes.NULLABLE_INTEGER.value,
54
+ AthenaDMLTypes.INT.value: PandasDtypes.NULLABLE_INTEGER.value,
55
+ AthenaDMLTypes.INTEGER.value: PandasDtypes.NULLABLE_INTEGER.value,
56
+ AthenaDMLTypes.BIGINT.value: PandasDtypes.NULLABLE_INTEGER.value,
57
+ AthenaDMLTypes.REAL.value: PandasDtypes.NULLABLE_FLOAT.value,
58
+ AthenaDMLTypes.DOUBLE.value: PandasDtypes.NULLABLE_FLOAT.value,
59
+ AthenaDMLTypes.DECIMAL.value: PandasDtypes.NULLABLE_FLOAT.value,
60
+ AthenaDMLTypes.CHAR.value: PandasDtypes.STRINGS.value,
61
+ AthenaDMLTypes.VARCHAR.value: PandasDtypes.STRINGS.value,
62
+ AthenaDMLTypes.VARBINARY.value: PandasDtypes.STRINGS.value,
63
+ AthenaDMLTypes.TIMESTAMP.value: PandasDtypes.DATETIME_NO_TZ.value,
64
+ AthenaDMLTypes.TIMESTAMP_WITH_TIME_ZONE.value: PandasDtypes.DATETIME_TZ_AWARE.value,
65
+ AthenaDMLTypes.ARRAY.value: PandasDtypes.OBJECT.value,
66
+ AthenaDMLTypes.MAP.value: PandasDtypes.OBJECT.value,
67
+ AthenaDMLTypes.ROW.value: PandasDtypes.OBJECT.value,
68
+ AthenaDMLTypes.JSON.value: PandasDtypes.OBJECT.value,
69
+ AthenaDMLTypes.UUID.value: PandasDtypes.STRINGS.value,
70
+ AthenaDMLTypes.IPADDRESS.value: PandasDtypes.STRINGS.value,
71
+ }