pal_tool 0.2.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (50) hide show
  1. checksums.yaml +7 -0
  2. data/.rspec +3 -0
  3. data/.rubocop.yml +132 -0
  4. data/CODE_OF_CONDUCT.md +84 -0
  5. data/Dockerfile +10 -0
  6. data/Gemfile +14 -0
  7. data/Gemfile.lock +72 -0
  8. data/LICENSE.txt +21 -0
  9. data/README.md +124 -0
  10. data/Rakefile +12 -0
  11. data/bin/console +15 -0
  12. data/bin/setup +8 -0
  13. data/exe/pal +47 -0
  14. data/lib/pal/common/local_file_utils.rb +37 -0
  15. data/lib/pal/common/object_helpers.rb +27 -0
  16. data/lib/pal/common/safe_hash_parse.rb +87 -0
  17. data/lib/pal/configuration.rb +77 -0
  18. data/lib/pal/handler/base.rb +138 -0
  19. data/lib/pal/handler/definitions/aws_cur.json +8 -0
  20. data/lib/pal/handler/manager.rb +30 -0
  21. data/lib/pal/handler/processor.rb +84 -0
  22. data/lib/pal/log.rb +29 -0
  23. data/lib/pal/main.rb +63 -0
  24. data/lib/pal/operation/actions.rb +106 -0
  25. data/lib/pal/operation/exporter.rb +183 -0
  26. data/lib/pal/operation/filter_evaluator.rb +249 -0
  27. data/lib/pal/operation/processor_context.rb +50 -0
  28. data/lib/pal/operation/projection.rb +302 -0
  29. data/lib/pal/plugin.rb +61 -0
  30. data/lib/pal/request/metadata.rb +19 -0
  31. data/lib/pal/request/runbook.rb +54 -0
  32. data/lib/pal/version.rb +5 -0
  33. data/lib/pal.rb +43 -0
  34. data/plugins/PLUGINS.md +1 -0
  35. data/plugins/operation/terminal_exporter_impl.rb +14 -0
  36. data/templates/DOCUMENTATION.md +46 -0
  37. data/templates/aws/data_transfer/data_transfer_breakdown.json +93 -0
  38. data/templates/aws/ec2/ec2_compute_hourly_breakdown.json +63 -0
  39. data/templates/aws/ec2/ec2_operation_breakdown.json +64 -0
  40. data/templates/aws/ec2/ec2_spend_breakdown.json +63 -0
  41. data/templates/aws/global_resource_and_usage_type_costs.json +41 -0
  42. data/templates/aws/kms/kms_usage_counts.json +52 -0
  43. data/templates/aws/kms/kms_usage_list.json +80 -0
  44. data/templates/aws/kms/list_of_kms_keys.json +57 -0
  45. data/templates/aws/reserved_instances/all_reserved_instance_expiries.json +41 -0
  46. data/templates/aws/reserved_instances/reserved_instance_opportunities.json +60 -0
  47. data/templates/aws/summary_cost_between_date_range.json +43 -0
  48. data/templates/aws/summary_daily_breakdown_costs.json +39 -0
  49. data/templates/azure/global_resource_type_summary.json +47 -0
  50. metadata +136 -0
@@ -0,0 +1,63 @@
1
+ {
2
+ "metadata" : {
3
+ "version" : "2022-04-02",
4
+ "name" : "EC2 Spend Breakdown",
5
+ "handler" : "AwsCur",
6
+ "description" : "Spend breakdown for EC2"
7
+ },
8
+ "filters": {
9
+ "condition": "AND",
10
+ "rules": [
11
+ {
12
+ "field": "lineItem/BlendedCost",
13
+ "type": "number",
14
+ "operator": "greater",
15
+ "value": 0
16
+ },{
17
+ "field": "lineItem/ProductCode",
18
+ "type": "string",
19
+ "operator": "equal",
20
+ "value": "AmazonEC2"
21
+ },{
22
+ "field": "lineItem/LineItemType",
23
+ "type": "string",
24
+ "operator": "not_equal",
25
+ "value": "Tax"
26
+ },{
27
+ "field": "lineItem/LineItemType",
28
+ "type": "string",
29
+ "operator": "not_equal",
30
+ "value": "Refund"
31
+ },{
32
+ "field": "lineItem/LineItemType",
33
+ "type": "string",
34
+ "operator": "not_equal",
35
+ "value": "Credit"
36
+ }
37
+ ]
38
+ },
39
+ "exporter" : {
40
+ "types" : [{
41
+ "name" : "table",
42
+ "settings" : {
43
+ "title" : "AWS CUR Resource/Usage Type Combined Costs"
44
+ }
45
+ }],
46
+ "properties" : [
47
+ "lineItem/UsageStartDate",
48
+ "lineItem/ProductCode",
49
+ "lineItem/LineItemDescription",
50
+ "lineItem/UnblendedCost"
51
+ ],
52
+ "actions" : {
53
+ "group_by" : ["lineItem/ProductCode", "lineItem/UsageStartDate"],
54
+ "sort_by" : "lineItem/UsageStartDate",
55
+ "projection" : {
56
+ "type" : "sum",
57
+ "property" : "lineItem/UnblendedCost"
58
+ }
59
+ }
60
+ }
61
+ }
62
+
63
+
@@ -0,0 +1,64 @@
1
+ {
2
+ "metadata" : {
3
+ "version" : "2022-04-02",
4
+ "name" : "EC2 Spend Breakdown",
5
+ "handler" : "AwsCur",
6
+ "description" : "Spend breakdown for EC2 family"
7
+ },
8
+ "filters": {
9
+ "condition": "AND",
10
+ "rules": [
11
+ {
12
+ "field": "lineItem/UnblendedCost",
13
+ "type": "number",
14
+ "operator": "greater",
15
+ "value": 0
16
+ },{
17
+ "field": "lineItem/ProductCode",
18
+ "type": "string",
19
+ "operator": "equal",
20
+ "value": "AmazonEC2"
21
+ },{
22
+ "field": "lineItem/LineItemType",
23
+ "type": "string",
24
+ "operator": "not_equal",
25
+ "value": "Tax"
26
+ },{
27
+ "field": "lineItem/LineItemType",
28
+ "type": "string",
29
+ "operator": "not_equal",
30
+ "value": "Refund"
31
+ },{
32
+ "field": "lineItem/LineItemType",
33
+ "type": "string",
34
+ "operator": "not_equal",
35
+ "value": "Credit"
36
+ }
37
+ ]
38
+ },
39
+ "exporter" : {
40
+ "types" : [{
41
+ "name" : "table",
42
+ "settings" : {
43
+ "title" : "AWS CUR Resource/Usage Type Combined Costs"
44
+ }
45
+ }],
46
+ "properties" : [
47
+ "lineItem/UsageStartDate",
48
+ "lineItem/ProductCode",
49
+ "lineItem/LineItemDescription",
50
+ "lineItem/UnblendedCost",
51
+ "lineItem/Operation"
52
+ ],
53
+ "actions" : {
54
+ "group_by" : ["lineItem/ProductCode", "lineItem/Operation"],
55
+ "sort_by" : "sum_lineItem/UnblendedCost",
56
+ "projection" : {
57
+ "type" : "sum",
58
+ "property" : "lineItem/UnblendedCost"
59
+ }
60
+ }
61
+ }
62
+ }
63
+
64
+
@@ -0,0 +1,63 @@
1
+ {
2
+ "metadata" : {
3
+ "version" : "2022-04-02",
4
+ "name" : "EC2 Spend Breakdown",
5
+ "handler" : "AwsCur",
6
+ "description" : "Spend breakdown for EC2 family"
7
+ },
8
+ "filters": {
9
+ "condition": "AND",
10
+ "rules": [
11
+ {
12
+ "field": "lineItem/BlendedCost",
13
+ "type": "number",
14
+ "operator": "greater",
15
+ "value": 0
16
+ },{
17
+ "field": "lineItem/ProductCode",
18
+ "type": "string",
19
+ "operator": "equal",
20
+ "value": "AmazonEC2"
21
+ },{
22
+ "field": "lineItem/LineItemType",
23
+ "type": "string",
24
+ "operator": "not_equal",
25
+ "value": "Tax"
26
+ },{
27
+ "field": "lineItem/LineItemType",
28
+ "type": "string",
29
+ "operator": "not_equal",
30
+ "value": "Refund"
31
+ },{
32
+ "field": "lineItem/LineItemType",
33
+ "type": "string",
34
+ "operator": "not_equal",
35
+ "value": "Credit"
36
+ }
37
+ ]
38
+ },
39
+ "exporter" : {
40
+ "types" : [{
41
+ "name" : "table",
42
+ "settings" : {
43
+ "title" : "AWS CUR Resource/Usage Type Combined Costs"
44
+ }
45
+ }],
46
+ "properties" : [
47
+ "lineItem/UsageStartDate",
48
+ "lineItem/ProductCode",
49
+ "lineItem/LineItemDescription",
50
+ "lineItem/UnblendedCost"
51
+ ],
52
+ "actions" : {
53
+ "group_by" : ["lineItem/ProductCode", "lineItem/LineItemDescription"],
54
+ "sort_by" : "sum_lineItem/UnblendedCost",
55
+ "projection" : {
56
+ "type" : "sum",
57
+ "property" : "lineItem/UnblendedCost"
58
+ }
59
+ }
60
+ }
61
+ }
62
+
63
+
@@ -0,0 +1,41 @@
1
+ {
2
+ "metadata" : {
3
+ "version" : "2022-04-02",
4
+ "name" : "Resource Type Breakdown Template",
5
+ "handler" : "AwsCur",
6
+ "description" : "Resource and usage type breakdown by cost."
7
+ },
8
+ "filters": {
9
+ "condition": "AND",
10
+ "rules": [{
11
+ "field": "lineItem/BlendedCost",
12
+ "type": "number",
13
+ "operator": "greater",
14
+ "value": 0
15
+ }
16
+ ]
17
+ },
18
+ "exporter" : {
19
+ "types" : [{
20
+ "name" : "table",
21
+ "settings" : {
22
+ "title" : "AWS CUR Resource/Usage Type Combined Costs"
23
+ }
24
+ }],
25
+ "properties" : [
26
+ "lineItem/UsageStartDate",
27
+ "lineItem/ProductCode",
28
+ "lineItem/UsageType",
29
+ "lineItem/BlendedCost",
30
+ "lineItem/UsageAccountId"
31
+ ],
32
+ "actions" : {
33
+ "group_by" : ["lineItem/ProductCode", "lineItem/UsageAccountId"],
34
+ "sort_by" : "sum_lineItem/BlendedCost",
35
+ "projection" : {
36
+ "type" : "sum",
37
+ "property" : "lineItem/BlendedCost"
38
+ }
39
+ }
40
+ }
41
+ }
@@ -0,0 +1,52 @@
1
+ {
2
+ "metadata" : {
3
+ "version" : "2022-04-02",
4
+ "name" : "KMS Report",
5
+ "handler" : "AwsCur",
6
+ "description" : "Usage counts for KMS"
7
+ },
8
+ "filters": {
9
+ "condition": "AND",
10
+ "rules": [{
11
+ "field": "lineItem/ProductCode",
12
+ "type": "string",
13
+ "operator": "equal",
14
+ "value": "awskms"
15
+ },{
16
+ "field": "lineItem/UsageType",
17
+ "type": "string",
18
+ "operator": "ends_with",
19
+ "value": "KMS-Requests"
20
+ },{
21
+ "field": "lineItem/ResourceId",
22
+ "type": "string",
23
+ "operator": "begins_with",
24
+ "value": "arn:aws:kms:"
25
+ }]
26
+ },
27
+ "exporter" : {
28
+ "types" : [{
29
+ "name" : "table",
30
+ "settings" : {
31
+ "title" : "Usage counts for KMS"
32
+ }
33
+ }],
34
+ "properties" : [
35
+ "lineItem/UsageType",
36
+ "lineItem/ProductCode",
37
+ "product/productFamily",
38
+ "lineItem/LineItemDescription",
39
+ "lineItem/ResourceId",
40
+ "lineItem/BlendedCost",
41
+ "lineItem/UsageAmount"
42
+ ],
43
+ "actions" : {
44
+ "group_by" : ["lineItem/ProductCode", "product/productFamily", "lineItem/ResourceId"],
45
+ "sort_by" : "sum_lineItem/UsageAmount",
46
+ "projection" : {
47
+ "type" : "sum",
48
+ "property" : "lineItem/UsageAmount"
49
+ }
50
+ }
51
+ }
52
+ }
@@ -0,0 +1,80 @@
1
+ {
2
+ "metadata" : {
3
+ "version" : "2022-04-02",
4
+ "name" : "KMS Report",
5
+ "handler" : "AwsCur",
6
+ "description" : "Usage counts for KMS"
7
+ },
8
+ "filters": {
9
+ "condition": "OR",
10
+ "rules": [
11
+ {
12
+ "condition": "AND",
13
+ "rules": [
14
+ {
15
+ "field": "lineItem/ProductCode",
16
+ "type": "string",
17
+ "operator": "equal",
18
+ "value": "awskms"
19
+ },
20
+ {
21
+ "field": "lineItem/UsageType",
22
+ "type": "string",
23
+ "operator": "ends_with",
24
+ "value": "KMS-Requests"
25
+ },
26
+ {
27
+ "field": "lineItem/ResourceId",
28
+ "type": "string",
29
+ "operator": "begins_with",
30
+ "value": "arn:aws:kms:"
31
+ }
32
+ ]
33
+ },
34
+ {
35
+ "condition": "AND",
36
+ "rules": [{
37
+ "field": "lineItem/BlendedCost",
38
+ "type": "number",
39
+ "operator": "greater",
40
+ "value": 0
41
+ },{
42
+ "field": "lineItem/ProductCode",
43
+ "type": "string",
44
+ "operator": "equal",
45
+ "value": "awskms"
46
+ },{
47
+ "field": "product/productFamily",
48
+ "type": "string",
49
+ "operator": "equal",
50
+ "value": "Encryption Key"
51
+ }]
52
+ }
53
+ ]
54
+ },
55
+ "exporter" : {
56
+ "types" : [{
57
+ "name" : "table",
58
+ "settings" : {
59
+ "title" : "Usage list for KMS"
60
+ }
61
+ }],
62
+ "properties" : [
63
+ "lineItem/UsageType",
64
+ "lineItem/ProductCode",
65
+ "product/productFamily",
66
+ "lineItem/LineItemDescription",
67
+ "lineItem/ResourceId",
68
+ "lineItem/BlendedCost",
69
+ "lineItem/UsageAmount"
70
+ ],
71
+ "actions" : {
72
+ "group_by" : ["lineItem/ProductCode", "product/productFamily", "lineItem/ResourceId"],
73
+ "sort_by" : "sum_lineItem/UsageAmount",
74
+ "projection" : {
75
+ "type" : "sum",
76
+ "property" : "lineItem/UsageAmount"
77
+ }
78
+ }
79
+ }
80
+ }
@@ -0,0 +1,57 @@
1
+ {
2
+ "metadata" : {
3
+ "version" : "2022-04-02",
4
+ "name" : "KMS Report",
5
+ "handler" : "AwsCur",
6
+ "description" : "List of billed KMS keys"
7
+ },
8
+ "filters": {
9
+ "condition": "AND",
10
+ "rules": [{
11
+ "field": "lineItem/BlendedCost",
12
+ "type": "number",
13
+ "operator": "greater",
14
+ "value": 0
15
+ },{
16
+ "field": "lineItem/ProductCode",
17
+ "type": "string",
18
+ "operator": "equal",
19
+ "value": "awskms"
20
+ },{
21
+ "field": "product/productFamily",
22
+ "type": "string",
23
+ "operator": "equal",
24
+ "value": "Encryption Key"
25
+ }]
26
+ },
27
+ "exporter" : {
28
+ "types" : [{
29
+ "name" : "csv",
30
+ "settings" : {
31
+ "file_name" : "pal_report_kms"
32
+ }
33
+ }, {
34
+ "name" : "table",
35
+ "settings" : {
36
+ "title" : "AWS CUR Daily Combined Costs"
37
+ }
38
+ }],
39
+ "properties" : [
40
+ "lineItem/UsageType",
41
+ "lineItem/ProductCode",
42
+ "product/productFamily",
43
+ "lineItem/LineItemDescription",
44
+ "lineItem/ResourceId",
45
+ "lineItem/BlendedCost",
46
+ "lineItem/UsageStartDate"
47
+ ],
48
+ "actions" : {
49
+ "group_by" : ["product/productFamily", "lineItem/ResourceId"],
50
+ "sort_by" : "distinct_lineItem/ResourceId",
51
+ "projection" : {
52
+ "type" : "distinct",
53
+ "property" : "lineItem/ResourceId"
54
+ }
55
+ }
56
+ }
57
+ }
@@ -0,0 +1,41 @@
1
+ {
2
+ "metadata" : {
3
+ "version" : "2022-04-02",
4
+ "name" : "RI Expiry Dates",
5
+ "handler" : "AwsCur",
6
+ "description" : "Reserved instance expiry dates"
7
+ },
8
+ "filters": {
9
+ "condition": "AND",
10
+ "rules": [{
11
+ "field": "reservation/EndTime",
12
+ "type": "string",
13
+ "operator": "is_not_empty",
14
+ "value": ""
15
+ }]
16
+ },
17
+ "exporter" : {
18
+ "types" : [{
19
+ "name" : "table",
20
+ "settings" : {
21
+ "title" : "Reserved instance expiry dates"
22
+ }
23
+ }],
24
+ "properties" : [
25
+ "lineItem/UsageType",
26
+ "lineItem/ProductCode",
27
+ "product/productFamily",
28
+ "lineItem/ResourceId",
29
+ "lineItem/BlendedCost",
30
+ "reservation/EndTime"
31
+ ],
32
+ "actions" : {
33
+ "group_by" : ["lineItem/ResourceId","reservation/EndTime"],
34
+ "sort_by" : "count_reservation/EndTime",
35
+ "projection" : {
36
+ "type" : "count",
37
+ "property" : "reservation/EndTime"
38
+ }
39
+ }
40
+ }
41
+ }
@@ -0,0 +1,60 @@
1
+ {
2
+ "metadata" : {
3
+ "version" : "2022-04-02",
4
+ "name" : "RI Opportunities",
5
+ "handler" : "AwsCur",
6
+ "description" : "Reserved Instance Opportunities"
7
+ },
8
+ "filters": {
9
+ "condition": "AND",
10
+ "rules": [{
11
+ "field": "lineItem/BlendedCost",
12
+ "type": "number",
13
+ "operator": "greater",
14
+ "value": 0
15
+ },{
16
+ "field": "lineItem/ProductCode",
17
+ "type": "string",
18
+ "operator": "equal",
19
+ "value": "AmazonEC2"
20
+ },{
21
+ "field": "lineItem/ResourceId",
22
+ "type": "string",
23
+ "operator": "begins_with",
24
+ "value": "i-"
25
+ }]
26
+ },
27
+ "exporter" : {
28
+ "types" : [{
29
+ "name" : "table",
30
+ "settings" : {
31
+ "title" : "AWS CUR Daily Combined Costs"
32
+ }
33
+ }],
34
+ "properties" : [
35
+ "lineItem/UsageType",
36
+ "lineItem/ProductCode",
37
+ "product/productFamily",
38
+ "lineItem/LineItemDescription",
39
+ "lineItem/ResourceId",
40
+ "lineItem/BlendedCost"
41
+ ],
42
+ "actions" : {
43
+ "group_by" : ["lineItem/ProductCode", "lineItem/UsageType", "product/productFamily", "lineItem/ResourceId", "lineItem/LineItemDescription"],
44
+ "sort_by" : "lineItem/ResourceId",
45
+ "projection" : {
46
+ "type" : "sum",
47
+ "property" : "lineItem/BlendedCost"
48
+ }
49
+ }
50
+ },
51
+ "__comments__" : [{
52
+ "notes" : "If filtering resource Ids, if a record doesn't have a *-BoxUsage:* usage type, it is RI'd",
53
+ "example_filter" : {
54
+ "field": "lineItem/UsageType",
55
+ "type": "string",
56
+ "operator": "contains",
57
+ "value": "-BoxUsage:"
58
+ }
59
+ }]
60
+ }
@@ -0,0 +1,43 @@
1
+ {
2
+ "metadata" : {
3
+ "version" : "2022-04-02",
4
+ "name" : "Summary Costs",
5
+ "handler" : "AwsCur",
6
+ "description" : "Summary costs between date range"
7
+ },
8
+ "filters": {
9
+ "condition": "AND",
10
+ "rules": [{
11
+ "field": "lineItem/UsageStartDate",
12
+ "type": "date",
13
+ "operator": "greater",
14
+ "value": "2022-03-15T00:00:00Z"
15
+ }]
16
+ },
17
+ "exporter" : {
18
+ "types" : [{
19
+ "name" : "table",
20
+ "settings" : {
21
+ "title" : "Summary costs between date range"
22
+ }
23
+ }],
24
+ "properties" : [
25
+ "lineItem/UsageStartDate",
26
+ "lineItem/UsageType",
27
+ "lineItem/BlendedCost"
28
+ ],
29
+ "actions" : {
30
+ "group_by" : ["lineItem/UsageStartDate"],
31
+ "sort_by" : "lineItem/UsageStartDate",
32
+ "projection" : {
33
+ "type" : "sum",
34
+ "property" : "lineItem/BlendedCost"
35
+ }
36
+ }
37
+ },
38
+ "column_overrides" : {
39
+ "lineItem/UsageStartDate" : {
40
+ "data_type": "date"
41
+ }
42
+ }
43
+ }
@@ -0,0 +1,39 @@
1
+ {
2
+ "metadata" : {
3
+ "version" : "2022-04-02",
4
+ "name" : "Resource Type Breakdown Template",
5
+ "handler" : "AwsCur",
6
+ "description" : "Resource and usage type breakdown by cost."
7
+ },
8
+ "filters": {
9
+ "condition": "AND",
10
+ "rules": [{
11
+ "field": "lineItem/BlendedCost",
12
+ "type": "number",
13
+ "operator": "greater",
14
+ "value": 0
15
+ }]
16
+ },
17
+ "exporter" : {
18
+ "types" : [{
19
+ "name" : "table",
20
+ "settings" : {
21
+ "title" : "AWS CUR Daily Combined Costs"
22
+ }
23
+ }],
24
+ "properties" : [
25
+ "lineItem/UsageStartDate",
26
+ "lineItem/ResourceId",
27
+ "lineItem/ProductCode",
28
+ "lineItem/BlendedCost"
29
+ ],
30
+ "actions" : {
31
+ "group_by" : ["lineItem/ResourceId", "lineItem/ProductCode"],
32
+ "sort_by" : "sum_lineItem/BlendedCost",
33
+ "projection" : {
34
+ "type" : "sum",
35
+ "property" : "lineItem/BlendedCost"
36
+ }
37
+ }
38
+ }
39
+ }
@@ -0,0 +1,47 @@
1
+ {
2
+ "metadata" : {
3
+ "version" : "2022-04-02",
4
+ "name" : "Spend breakdown by Resource Type",
5
+ "handler" : "GenericCSV",
6
+ "description" : "Spend breakdown by Resource Type"
7
+ },
8
+ "filters": {
9
+ "condition": "AND",
10
+ "rules": [
11
+ {
12
+ "field": "PreTaxCost",
13
+ "type": "number",
14
+ "operator": "greater",
15
+ "value": 0
16
+ }]
17
+ },
18
+ "exporter" : {
19
+ "types" : [{
20
+ "name" : "table",
21
+ "settings" : {
22
+ "title" : "Spend breakdown by Resource Type"
23
+ }
24
+ }],
25
+ "properties" : [
26
+ "ResourceType",
27
+ "PreTaxCost"
28
+ ],
29
+ "actions" : {
30
+ "group_by" : ["ResourceType"],
31
+ "sort_by" : "sum_PreTaxCost",
32
+ "projection" : {
33
+ "type" : "sum",
34
+ "property" : "PreTaxCost"
35
+ }
36
+ }
37
+ },
38
+ "transforms" : [
39
+ {
40
+ "column": "Tags",
41
+ "strategy" : "json",
42
+ "options" : {}
43
+ }
44
+ ]
45
+ }
46
+
47
+