modularization_statistics 1.33.0 → 1.36.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +55 -0
- data/lib/modularization_statistics/private/datadog_reporter.rb +13 -278
- data/lib/modularization_statistics/private/metrics/files.rb +47 -0
- data/lib/modularization_statistics/private/metrics/nested_packs.rb +137 -0
- data/lib/modularization_statistics/private/metrics/packages.rb +109 -0
- data/lib/modularization_statistics/private/metrics/packages_by_team.rb +73 -0
- data/lib/modularization_statistics/private/metrics/protection_usage.rb +33 -0
- data/lib/modularization_statistics/private/metrics/public_usage.rb +36 -0
- data/lib/modularization_statistics/private/metrics.rb +49 -0
- data/lib/modularization_statistics/private/source_code_file.rb +1 -1
- data/lib/modularization_statistics.rb +1 -1
- data/sorbet/rbi/gems/{code_ownership@1.23.0.rbi → code_ownership@1.28.0.rbi} +27 -26
- data/sorbet/rbi/gems/{bigrails-teams@0.1.0.rbi → code_teams@1.0.0.rbi} +25 -25
- data/sorbet/rbi/gems/{package_protections@0.64.0.rbi → package_protections@1.4.0.rbi} +56 -107
- data/sorbet/rbi/gems/{parse_packwerk@0.10.0.rbi → parse_packwerk@0.12.0.rbi} +6 -0
- metadata +18 -11
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3684ee2e07b5f3cba3bb7093be91b88725808a8a3d203e4ede46c7aa116d920c
|
4
|
+
data.tar.gz: 7872ee6e03ff44a158e2274861c0de2700227e976078e3c905dc8f102d58cfaa
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c6d080bc0c7e9c199e37262c8949a9227510eae2e7cac98d8c90121f788aca0b5123da006ba4a3ade27b512b3a7373ea06f789f82d108d51dc6d83cdf3378291
|
7
|
+
data.tar.gz: 3333265a294391652a0d07199c897e6ff02f2e40536e9a3654f7cfa2c21d1bf91c4f6cff0d5771eeea581b3f17740621d317c7f2a4a058960fd1328a9a801457
|
data/README.md
CHANGED
@@ -43,6 +43,61 @@ ModularizationStatistics.report_to_datadog!(
|
|
43
43
|
)
|
44
44
|
```
|
45
45
|
|
46
|
+
It's recommended to run this in CI on the main/development branch so each new commit has metrics emitted for it.
|
47
|
+
|
48
|
+
# Tracking Privacy and Dependency Violations Reliably
|
49
|
+
With [`packwerk`](https://github.com/Shopify/packwerk), privacy and dependency violations do not show up until a package has set `enforce_privacy` and `enforce_dependency` (respectively) to `true`. As such, when you're first starting off, you'll see no violations, and then periodic large increases as teams start using these protections. If you're interested in looking at privacy and dependency violations over time as if all packages were enforcing dependencies and privacy the whole time, we recommend setting these values to be true before running modularization statistics in your CI.
|
50
|
+
|
51
|
+
```ruby
|
52
|
+
require 'modularization_statistics'
|
53
|
+
|
54
|
+
namespace(:modularization) do
|
55
|
+
desc(
|
56
|
+
'Publish modularization stats to datadog. ' \
|
57
|
+
'Example: bin/rails "modularization:upload_statistics"'
|
58
|
+
)
|
59
|
+
task(:upload_statistics, [:verbose] => :environment) do |_, args|
|
60
|
+
ignored_paths = Pathname.glob('spec/fixtures/**/**')
|
61
|
+
source_code_pathnames = Pathname.glob('{app,components,lib,packs,spec}/**/**').select(&:file?) - ignored_paths
|
62
|
+
|
63
|
+
# To correctly track violations, we rewrite all `package.yml` files with
|
64
|
+
# `enforce_dependencies` and `enforce_privacy` set to true, then update deprecations.
|
65
|
+
old_packages = ParsePackwerk.all
|
66
|
+
old_packages.each do |package|
|
67
|
+
new_package = ParsePackwerk::Package.new(
|
68
|
+
dependencies: package.dependencies,
|
69
|
+
enforce_dependencies: true,
|
70
|
+
enforce_privacy: true,
|
71
|
+
metadata: package.metadata,
|
72
|
+
name: package.name
|
73
|
+
)
|
74
|
+
ParsePackwerk.write_package_yml!(new_package)
|
75
|
+
end
|
76
|
+
|
77
|
+
Packwerk::Cli.new.execute_command(['update-deprecations'])
|
78
|
+
|
79
|
+
# Now we reset it back so that the protection values are the same as the native packwerk configuration
|
80
|
+
old_packages.each do |package|
|
81
|
+
new_package = ParsePackwerk::Package.new(
|
82
|
+
dependencies: package.dependencies,
|
83
|
+
enforce_dependencies: package.enforce_dependencies,
|
84
|
+
enforce_privacy: package.enforce_privacy,
|
85
|
+
metadata: package.metadata,
|
86
|
+
name: package.name
|
87
|
+
)
|
88
|
+
ParsePackwerk.write_package_yml!(new_package)
|
89
|
+
end
|
90
|
+
|
91
|
+
ModularizationStatistics.report_to_datadog!(
|
92
|
+
datadog_client: Dogapi::Client.new(ENV.fetch('DATADOG_API_KEY')),
|
93
|
+
app_name: Rails.application.class.module_parent_name,
|
94
|
+
source_code_pathnames: source_code_pathnames,
|
95
|
+
verbose: args[:verbose] == 'true' || false
|
96
|
+
)
|
97
|
+
end
|
98
|
+
end
|
99
|
+
```
|
100
|
+
|
46
101
|
# Using Other Observability Tools
|
47
102
|
|
48
103
|
Right now this tool sends metrics to DataDog early. However, if you want to use this with other tools, you can call `ModularizationStatistics.get_metrics(...)` to get generic metrics that you can then send to whatever observability provider you use.
|
@@ -2,14 +2,19 @@
|
|
2
2
|
# frozen_string_literal: true
|
3
3
|
|
4
4
|
require 'dogapi'
|
5
|
+
require 'modularization_statistics/private/metrics'
|
6
|
+
require 'modularization_statistics/private/metrics/files'
|
7
|
+
require 'modularization_statistics/private/metrics/public_usage'
|
8
|
+
require 'modularization_statistics/private/metrics/protection_usage'
|
9
|
+
require 'modularization_statistics/private/metrics/packages'
|
10
|
+
require 'modularization_statistics/private/metrics/packages_by_team'
|
11
|
+
require 'modularization_statistics/private/metrics/nested_packs'
|
5
12
|
|
6
13
|
module ModularizationStatistics
|
7
14
|
module Private
|
8
15
|
class DatadogReporter
|
9
16
|
extend T::Sig
|
10
17
|
|
11
|
-
UNKNOWN_OWNER = T.let('Unknown', String)
|
12
|
-
|
13
18
|
sig do
|
14
19
|
params(
|
15
20
|
source_code_files: T::Array[SourceCodeFile],
|
@@ -17,22 +22,14 @@ module ModularizationStatistics
|
|
17
22
|
).returns(T::Array[GaugeMetric])
|
18
23
|
end
|
19
24
|
def self.get_metrics(source_code_files:, app_name:)
|
20
|
-
all_metrics = T.let([], T::Array[GaugeMetric])
|
21
|
-
app_level_tag = Tag.for('app', app_name)
|
22
|
-
|
23
|
-
source_code_files.group_by { |file| file.team_owner&.name }.each do |team_name, files_for_team|
|
24
|
-
file_tags = tags_for_team(team_name) + [app_level_tag]
|
25
|
-
all_metrics += get_file_metrics('by_team', file_tags, files_for_team)
|
26
|
-
end
|
27
|
-
|
28
|
-
file_tags = [app_level_tag]
|
29
|
-
all_metrics += get_file_metrics('totals', file_tags, source_code_files)
|
30
|
-
|
31
25
|
packages = ParsePackwerk.all
|
32
|
-
all_metrics += get_package_metrics(packages, app_name)
|
33
|
-
all_metrics += get_package_metrics_by_team(packages, app_name)
|
34
26
|
|
35
|
-
|
27
|
+
[
|
28
|
+
*Metrics::Files.get_metrics(source_code_files, app_name),
|
29
|
+
*Metrics::Packages.get_package_metrics(packages, app_name),
|
30
|
+
*Metrics::PackagesByTeam.get_package_metrics_by_team(packages, app_name),
|
31
|
+
*Metrics::NestedPacks.get_nested_package_metrics(packages, app_name)
|
32
|
+
]
|
36
33
|
end
|
37
34
|
|
38
35
|
sig do
|
@@ -58,268 +55,6 @@ module ModularizationStatistics
|
|
58
55
|
end
|
59
56
|
end
|
60
57
|
end
|
61
|
-
|
62
|
-
sig { params(package: ParsePackwerk::Package, app_name: String).returns(T::Array[Tag]) }
|
63
|
-
def self.tags_for_package(package, app_name)
|
64
|
-
[
|
65
|
-
Tag.new(key: 'package', value: humanized_package_name(package.name)),
|
66
|
-
Tag.new(key: 'app', value: app_name),
|
67
|
-
*tags_for_team(CodeOwnership.for_package(package)&.name),
|
68
|
-
]
|
69
|
-
end
|
70
|
-
|
71
|
-
sig { params(team_name: T.nilable(String)).returns(T::Array[Tag]) }
|
72
|
-
def self.tags_for_team(team_name)
|
73
|
-
[Tag.for('team', team_name || UNKNOWN_OWNER)]
|
74
|
-
end
|
75
|
-
|
76
|
-
sig { params(team_name: T.nilable(String)).returns(T::Array[Tag]) }
|
77
|
-
def self.tags_for_to_team(team_name)
|
78
|
-
[Tag.for('to_team', team_name || UNKNOWN_OWNER)]
|
79
|
-
end
|
80
|
-
|
81
|
-
private_class_method :tags_for_package
|
82
|
-
|
83
|
-
sig do
|
84
|
-
params(
|
85
|
-
packages: T::Array[ParsePackwerk::Package],
|
86
|
-
app_name: String
|
87
|
-
).returns(T::Array[GaugeMetric])
|
88
|
-
end
|
89
|
-
def self.get_package_metrics(packages, app_name)
|
90
|
-
all_metrics = []
|
91
|
-
app_level_tag = Tag.for('app', app_name)
|
92
|
-
package_tags = T.let([app_level_tag], T::Array[Tag])
|
93
|
-
protected_packages = packages.map { |p| PackageProtections::ProtectedPackage.from(p) }
|
94
|
-
|
95
|
-
all_metrics << GaugeMetric.for('all_packages.count', packages.count, package_tags)
|
96
|
-
all_metrics << GaugeMetric.for('all_packages.dependencies.count', packages.sum { |package| package.dependencies.count }, package_tags)
|
97
|
-
all_metrics << GaugeMetric.for('all_packages.dependency_violations.count', protected_packages.sum { |package| file_count(package.violations.select(&:dependency?)) }, package_tags)
|
98
|
-
all_metrics << GaugeMetric.for('all_packages.privacy_violations.count', protected_packages.sum { |package| file_count(package.violations.select(&:privacy?)) }, package_tags)
|
99
|
-
all_metrics << GaugeMetric.for('all_packages.enforcing_dependencies.count', packages.count(&:enforces_dependencies?), package_tags)
|
100
|
-
all_metrics << GaugeMetric.for('all_packages.enforcing_privacy.count', packages.count(&:enforces_privacy?), package_tags)
|
101
|
-
|
102
|
-
all_metrics << GaugeMetric.for('all_packages.notify_on_package_yml_changes.count', packages.count { |p| p.metadata['notify_on_package_yml_changes'] }, package_tags)
|
103
|
-
all_metrics << GaugeMetric.for('all_packages.notify_on_new_violations.count', packages.count { |p| p.metadata['notify_on_new_violations'] }, package_tags)
|
104
|
-
|
105
|
-
all_metrics << GaugeMetric.for('all_packages.with_violations.count', protected_packages.count { |package| package.violations.any? }, package_tags)
|
106
|
-
all_metrics += self.get_public_usage_metrics('all_packages', packages, package_tags)
|
107
|
-
all_metrics << GaugeMetric.for('all_packages.has_readme.count', packages.count { |package| has_readme?(package) }, package_tags)
|
108
|
-
|
109
|
-
all_metrics += self.get_protections_metrics('all_packages', protected_packages, package_tags)
|
110
|
-
all_metrics << GaugeMetric.for('all_packages.package_based_file_ownership.count', packages.count { |package| !package.metadata['owner'].nil? }, package_tags)
|
111
|
-
|
112
|
-
inbound_violations_by_package = protected_packages.flat_map(&:violations).group_by(&:to_package_name)
|
113
|
-
|
114
|
-
protected_packages.each do |protected_package|
|
115
|
-
package = protected_package.original_package
|
116
|
-
package_tags = tags_for_package(package, app_name)
|
117
|
-
|
118
|
-
#
|
119
|
-
# VIOLATIONS (implicit dependencies)
|
120
|
-
#
|
121
|
-
outbound_violations = protected_package.violations
|
122
|
-
inbound_violations = inbound_violations_by_package[package.name] || []
|
123
|
-
all_dependency_violations = (outbound_violations + inbound_violations).select(&:dependency?)
|
124
|
-
all_privacy_violations = (outbound_violations + inbound_violations).select(&:privacy?)
|
125
|
-
|
126
|
-
all_metrics << GaugeMetric.for('by_package.dependency_violations.count', file_count(all_dependency_violations), package_tags)
|
127
|
-
all_metrics << GaugeMetric.for('by_package.privacy_violations.count', file_count(all_privacy_violations), package_tags)
|
128
|
-
|
129
|
-
all_metrics << GaugeMetric.for('by_package.outbound_dependency_violations.count', file_count(outbound_violations.select(&:dependency?)), package_tags)
|
130
|
-
all_metrics << GaugeMetric.for('by_package.inbound_dependency_violations.count', file_count(inbound_violations.select(&:dependency?)), package_tags)
|
131
|
-
|
132
|
-
all_metrics << GaugeMetric.for('by_package.outbound_privacy_violations.count', file_count(outbound_violations.select(&:privacy?)), package_tags)
|
133
|
-
all_metrics << GaugeMetric.for('by_package.inbound_privacy_violations.count', file_count(inbound_violations.select(&:privacy?)), package_tags)
|
134
|
-
|
135
|
-
all_metrics += self.get_public_usage_metrics('by_package', [package], package_tags)
|
136
|
-
|
137
|
-
protected_package.violations.group_by(&:to_package_name).each do |to_package_name, violations|
|
138
|
-
to_package = ParsePackwerk.find(to_package_name)
|
139
|
-
if to_package.nil?
|
140
|
-
raise StandardError, "Could not find matching package #{to_package_name}"
|
141
|
-
end
|
142
|
-
|
143
|
-
tags = package_tags + [Tag.for('to_package', humanized_package_name(to_package_name))] + tags_for_to_team(CodeOwnership.for_package(to_package)&.name)
|
144
|
-
all_metrics << GaugeMetric.for('by_package.outbound_dependency_violations.per_package.count', file_count(violations.select(&:dependency?)), tags)
|
145
|
-
all_metrics << GaugeMetric.for('by_package.outbound_privacy_violations.per_package.count', file_count(violations.select(&:privacy?)), tags)
|
146
|
-
end
|
147
|
-
end
|
148
|
-
|
149
|
-
inbound_explicit_dependency_by_package = {}
|
150
|
-
packages.each do |package|
|
151
|
-
package.dependencies.each do |explicit_dependency|
|
152
|
-
inbound_explicit_dependency_by_package[explicit_dependency] ||= []
|
153
|
-
inbound_explicit_dependency_by_package[explicit_dependency] << package.name
|
154
|
-
end
|
155
|
-
end
|
156
|
-
|
157
|
-
packages.each do |package| # rubocop:disable Style/CombinableLoops
|
158
|
-
package_tags = tags_for_package(package, app_name)
|
159
|
-
|
160
|
-
#
|
161
|
-
# EXPLICIT DEPENDENCIES
|
162
|
-
#
|
163
|
-
package.dependencies.each do |explicit_dependency|
|
164
|
-
to_package = ParsePackwerk.find(explicit_dependency)
|
165
|
-
if to_package.nil?
|
166
|
-
raise StandardError, "Could not find matching package #{explicit_dependency}"
|
167
|
-
end
|
168
|
-
|
169
|
-
tags = package_tags + [Tag.for('to_package', humanized_package_name(explicit_dependency))] + tags_for_to_team(CodeOwnership.for_package(to_package)&.name)
|
170
|
-
all_metrics << GaugeMetric.for('by_package.outbound_explicit_dependencies.per_package.count', 1, tags)
|
171
|
-
end
|
172
|
-
|
173
|
-
all_metrics << GaugeMetric.for('by_package.outbound_explicit_dependencies.count', package.dependencies.count, package_tags)
|
174
|
-
all_metrics << GaugeMetric.for('by_package.inbound_explicit_dependencies.count', inbound_explicit_dependency_by_package[package.name]&.count || 0, package_tags)
|
175
|
-
end
|
176
|
-
|
177
|
-
all_metrics
|
178
|
-
end
|
179
|
-
|
180
|
-
sig do
|
181
|
-
params(
|
182
|
-
all_packages: T::Array[ParsePackwerk::Package],
|
183
|
-
app_name: String
|
184
|
-
).returns(T::Array[GaugeMetric])
|
185
|
-
end
|
186
|
-
def self.get_package_metrics_by_team(all_packages, app_name)
|
187
|
-
all_metrics = T.let([], T::Array[GaugeMetric])
|
188
|
-
app_level_tag = Tag.for('app', app_name)
|
189
|
-
all_protected_packages = all_packages.map { |p| PackageProtections::ProtectedPackage.from(p) }
|
190
|
-
all_protected_packages.group_by { |protected_package| CodeOwnership.for_package(protected_package.original_package)&.name }.each do |team_name, protected_packages_by_team|
|
191
|
-
# We look at `all_packages` because we care about ALL inbound violations across all teams
|
192
|
-
inbound_violations_by_package = all_protected_packages.flat_map(&:violations).group_by(&:to_package_name)
|
193
|
-
|
194
|
-
team_tags = tags_for_team(team_name) + [app_level_tag]
|
195
|
-
all_metrics << GaugeMetric.for('by_team.all_packages.count', protected_packages_by_team.count, team_tags)
|
196
|
-
all_metrics += self.get_protections_metrics('by_team', protected_packages_by_team, team_tags)
|
197
|
-
all_metrics += self.get_public_usage_metrics('by_team', protected_packages_by_team.map(&:original_package), team_tags)
|
198
|
-
|
199
|
-
all_metrics << GaugeMetric.for('by_team.notify_on_package_yml_changes.count', protected_packages_by_team.count { |p| p.metadata['notify_on_package_yml_changes'] }, team_tags)
|
200
|
-
all_metrics << GaugeMetric.for('by_team.notify_on_new_violations.count', protected_packages_by_team.count { |p| p.metadata['notify_on_new_violations'] }, team_tags)
|
201
|
-
|
202
|
-
#
|
203
|
-
# VIOLATIONS (implicit dependencies)
|
204
|
-
#
|
205
|
-
outbound_violations = protected_packages_by_team.flat_map(&:violations)
|
206
|
-
# Here we only look at packages_by_team because we only care about inbound violations onto packages for this team
|
207
|
-
inbound_violations = protected_packages_by_team.flat_map { |package| inbound_violations_by_package[package.name] || [] }
|
208
|
-
all_dependency_violations = (outbound_violations + inbound_violations).select(&:dependency?)
|
209
|
-
all_privacy_violations = (outbound_violations + inbound_violations).select(&:privacy?)
|
210
|
-
|
211
|
-
all_metrics << GaugeMetric.for('by_team.dependency_violations.count', file_count(all_dependency_violations), team_tags)
|
212
|
-
all_metrics << GaugeMetric.for('by_team.privacy_violations.count', file_count(all_privacy_violations), team_tags)
|
213
|
-
|
214
|
-
all_metrics << GaugeMetric.for('by_team.outbound_dependency_violations.count', file_count(outbound_violations.select(&:dependency?)), team_tags)
|
215
|
-
all_metrics << GaugeMetric.for('by_team.inbound_dependency_violations.count', file_count(inbound_violations.select(&:dependency?)), team_tags)
|
216
|
-
|
217
|
-
all_metrics << GaugeMetric.for('by_team.outbound_privacy_violations.count', file_count(outbound_violations.select(&:privacy?)), team_tags)
|
218
|
-
all_metrics << GaugeMetric.for('by_team.inbound_privacy_violations.count', file_count(inbound_violations.select(&:privacy?)), team_tags)
|
219
|
-
|
220
|
-
all_metrics << GaugeMetric.for('by_team.has_readme.count', protected_packages_by_team.count { |protected_package| has_readme?(protected_package.original_package) }, team_tags)
|
221
|
-
|
222
|
-
grouped_outbound_violations = outbound_violations.group_by do |violation|
|
223
|
-
to_package = ParsePackwerk.find(violation.to_package_name)
|
224
|
-
if to_package.nil?
|
225
|
-
raise StandardError, "Could not find matching package #{violation.to_package_name}"
|
226
|
-
end
|
227
|
-
|
228
|
-
CodeOwnership.for_package(to_package)&.name
|
229
|
-
end
|
230
|
-
|
231
|
-
grouped_outbound_violations.each do |to_team_name, violations|
|
232
|
-
tags = team_tags + tags_for_to_team(to_team_name)
|
233
|
-
all_metrics << GaugeMetric.for('by_team.outbound_dependency_violations.per_team.count', file_count(violations.select(&:dependency?)), tags)
|
234
|
-
all_metrics << GaugeMetric.for('by_team.outbound_privacy_violations.per_team.count', file_count(violations.select(&:privacy?)), tags)
|
235
|
-
end
|
236
|
-
end
|
237
|
-
|
238
|
-
all_metrics
|
239
|
-
end
|
240
|
-
|
241
|
-
private_class_method :get_package_metrics
|
242
|
-
|
243
|
-
sig do
|
244
|
-
params(
|
245
|
-
metric_name_suffix: String,
|
246
|
-
tags: T::Array[Tag],
|
247
|
-
files: T::Array[SourceCodeFile]
|
248
|
-
).returns(T::Array[GaugeMetric])
|
249
|
-
end
|
250
|
-
def self.get_file_metrics(metric_name_suffix, tags, files)
|
251
|
-
[
|
252
|
-
GaugeMetric.for("component_files.#{metric_name_suffix}", files.count(&:componentized_file?), tags),
|
253
|
-
GaugeMetric.for("packaged_files.#{metric_name_suffix}", files.count(&:packaged_file?), tags),
|
254
|
-
GaugeMetric.for("all_files.#{metric_name_suffix}", files.count, tags),
|
255
|
-
]
|
256
|
-
end
|
257
|
-
|
258
|
-
private_class_method :get_file_metrics
|
259
|
-
|
260
|
-
sig { params(prefix: String, protected_packages: T::Array[PackageProtections::ProtectedPackage], package_tags: T::Array[Tag]).returns(T::Array[GaugeMetric]) }
|
261
|
-
def self.get_protections_metrics(prefix, protected_packages, package_tags)
|
262
|
-
PackageProtections.all.flat_map do |protection|
|
263
|
-
PackageProtections::ViolationBehavior.each_value.map do |violation_behavior|
|
264
|
-
# https://github.com/Gusto/package_protections/pull/42 changed the public API of these violation behaviors.
|
265
|
-
# To preserve our ability to understand historical trends, we map to the old values.
|
266
|
-
# This allows our dashboards to continue to operate as expected.
|
267
|
-
# Note if we ever open source mod stats, we should probably inject this behavior so that new clients can see the new keys in their metrics.
|
268
|
-
violation_behavior_map = {
|
269
|
-
PackageProtections::ViolationBehavior::FailOnAny => 'fail_the_build_on_any_instances',
|
270
|
-
PackageProtections::ViolationBehavior::FailNever => 'no',
|
271
|
-
PackageProtections::ViolationBehavior::FailOnNew => 'fail_the_build_if_new_instances_appear',
|
272
|
-
}
|
273
|
-
violation_behavior_name = violation_behavior_map[violation_behavior]
|
274
|
-
metric_name = "#{prefix}.#{protection.identifier}.#{violation_behavior_name}.count"
|
275
|
-
count_of_packages = protected_packages.count { |p| p.violation_behavior_for(protection.identifier) == violation_behavior }
|
276
|
-
GaugeMetric.for(metric_name, count_of_packages, package_tags)
|
277
|
-
end
|
278
|
-
end
|
279
|
-
end
|
280
|
-
|
281
|
-
sig { params(prefix: String, packages: T::Array[ParsePackwerk::Package], package_tags: T::Array[Tag]).returns(T::Array[GaugeMetric]) }
|
282
|
-
def self.get_public_usage_metrics(prefix, packages, package_tags)
|
283
|
-
packages_except_for_root = packages.reject { |package| package.name == ParsePackwerk::ROOT_PACKAGE_NAME }
|
284
|
-
all_files = packages_except_for_root.flat_map do |package|
|
285
|
-
package.directory.glob('**/**.rb')
|
286
|
-
end
|
287
|
-
|
288
|
-
all_public_files = T.let([], T::Array[Pathname])
|
289
|
-
is_using_public_directory = 0
|
290
|
-
packages_except_for_root.each do |package|
|
291
|
-
public_files = package.directory.glob('app/public/**/**.rb')
|
292
|
-
all_public_files += public_files
|
293
|
-
is_using_public_directory += 1 if public_files.any?
|
294
|
-
end
|
295
|
-
|
296
|
-
# In Datadog, can divide public files by all files to get the ratio.
|
297
|
-
# This is not a metric that we are targeting -- its for observability and reflection only.
|
298
|
-
[
|
299
|
-
GaugeMetric.for("#{prefix}.all_files.count", all_files.count, package_tags),
|
300
|
-
GaugeMetric.for("#{prefix}.public_files.count", all_public_files.count, package_tags),
|
301
|
-
GaugeMetric.for("#{prefix}.using_public_directory.count", is_using_public_directory, package_tags),
|
302
|
-
]
|
303
|
-
end
|
304
|
-
|
305
|
-
sig { params(package: ParsePackwerk::Package).returns(T::Boolean) }
|
306
|
-
def self.has_readme?(package)
|
307
|
-
package.directory.join('README.md').exist?
|
308
|
-
end
|
309
|
-
|
310
|
-
sig { params(violations: T::Array[ParsePackwerk::Violation]).returns(Integer) }
|
311
|
-
def self.file_count(violations)
|
312
|
-
violations.sum { |v| v.files.count }
|
313
|
-
end
|
314
|
-
|
315
|
-
sig { params(name: String).returns(String) }
|
316
|
-
def self.humanized_package_name(name)
|
317
|
-
if name == ParsePackwerk::ROOT_PACKAGE_NAME
|
318
|
-
'root'
|
319
|
-
else
|
320
|
-
name
|
321
|
-
end
|
322
|
-
end
|
323
58
|
end
|
324
59
|
end
|
325
60
|
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
# typed: strict
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
module ModularizationStatistics
|
5
|
+
module Private
|
6
|
+
module Metrics
|
7
|
+
class Files
|
8
|
+
extend T::Sig
|
9
|
+
|
10
|
+
sig do
|
11
|
+
params(
|
12
|
+
source_code_files: T::Array[SourceCodeFile],
|
13
|
+
app_name: String
|
14
|
+
).returns(T::Array[GaugeMetric])
|
15
|
+
end
|
16
|
+
def self.get_metrics(source_code_files, app_name)
|
17
|
+
all_metrics = T.let([], T::Array[GaugeMetric])
|
18
|
+
app_level_tag = Tag.for('app', app_name)
|
19
|
+
|
20
|
+
source_code_files.group_by { |file| file.team_owner&.name }.each do |team_name, files_for_team|
|
21
|
+
file_tags = Metrics.tags_for_team(team_name) + [app_level_tag]
|
22
|
+
all_metrics += get_file_metrics('by_team', file_tags, files_for_team)
|
23
|
+
end
|
24
|
+
|
25
|
+
file_tags = [app_level_tag]
|
26
|
+
all_metrics += get_file_metrics('totals', file_tags, source_code_files)
|
27
|
+
all_metrics
|
28
|
+
end
|
29
|
+
|
30
|
+
sig do
|
31
|
+
params(
|
32
|
+
metric_name_suffix: String,
|
33
|
+
tags: T::Array[Tag],
|
34
|
+
files: T::Array[SourceCodeFile]
|
35
|
+
).returns(T::Array[GaugeMetric])
|
36
|
+
end
|
37
|
+
def self.get_file_metrics(metric_name_suffix, tags, files)
|
38
|
+
[
|
39
|
+
GaugeMetric.for("component_files.#{metric_name_suffix}", files.count(&:componentized_file?), tags),
|
40
|
+
GaugeMetric.for("packaged_files.#{metric_name_suffix}", files.count(&:packaged_file?), tags),
|
41
|
+
GaugeMetric.for("all_files.#{metric_name_suffix}", files.count, tags),
|
42
|
+
]
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,137 @@
|
|
1
|
+
# typed: strict
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
module ModularizationStatistics
|
5
|
+
module Private
|
6
|
+
module Metrics
|
7
|
+
class NestedPacks
|
8
|
+
extend T::Sig
|
9
|
+
|
10
|
+
class PackGroup < T::Struct
|
11
|
+
extend T::Sig
|
12
|
+
|
13
|
+
const :name, String
|
14
|
+
const :root, ParsePackwerk::Package
|
15
|
+
const :members, T::Array[ParsePackwerk::Package]
|
16
|
+
|
17
|
+
sig { params(packages: T::Array[ParsePackwerk::Package]).returns(T::Array[PackGroup]) }
|
18
|
+
def self.all_from(packages)
|
19
|
+
packs_by_group = {}
|
20
|
+
|
21
|
+
packages.each do |package|
|
22
|
+
# For a child pack, package.directory is `packs/fruits/apples` (i.e. the directory of the package.yml file).
|
23
|
+
# The package.directory.dirname is therefore `packs/fruits`.
|
24
|
+
# For a standalone pack, package.directory.dirname is `packs`
|
25
|
+
# A pack with no parent is in a pack group of its own name
|
26
|
+
root = ParsePackwerk.find(package.directory.dirname.to_s) || package
|
27
|
+
# Mark the parent pack and child pack as being in the pack group of the parent
|
28
|
+
packs_by_group[root.name] ||= { root: root, members: [] }
|
29
|
+
packs_by_group[root.name][:members] << package
|
30
|
+
end
|
31
|
+
|
32
|
+
packs_by_group.map do |name, pack_data|
|
33
|
+
PackGroup.new(
|
34
|
+
name: name,
|
35
|
+
root: pack_data[:root],
|
36
|
+
members: pack_data[:members],
|
37
|
+
)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
sig { returns(Integer) }
|
42
|
+
def children_pack_count
|
43
|
+
members.count do |package|
|
44
|
+
package.name != root.name
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
sig { returns(T::Boolean) }
|
49
|
+
def has_parent?
|
50
|
+
children_pack_count > 0
|
51
|
+
end
|
52
|
+
|
53
|
+
sig { returns(T::Array[ParsePackwerk::Violation]) }
|
54
|
+
def cross_group_violations
|
55
|
+
all_violations = members.flat_map do |member|
|
56
|
+
ParsePackwerk::DeprecatedReferences.for(member).violations
|
57
|
+
end
|
58
|
+
|
59
|
+
all_violations.select do |violation|
|
60
|
+
!members.map(&:name).include?(violation.to_package_name)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
sig do
|
66
|
+
params(
|
67
|
+
packages: T::Array[ParsePackwerk::Package],
|
68
|
+
app_name: String
|
69
|
+
).returns(T::Array[GaugeMetric])
|
70
|
+
end
|
71
|
+
def self.get_nested_package_metrics(packages, app_name)
|
72
|
+
all_metrics = []
|
73
|
+
app_level_tag = Tag.for('app', app_name)
|
74
|
+
package_tags = T.let([app_level_tag], T::Array[Tag])
|
75
|
+
|
76
|
+
pack_groups = PackGroup.all_from(packages)
|
77
|
+
all_pack_groups_count = pack_groups.count
|
78
|
+
child_pack_count = pack_groups.sum(&:children_pack_count)
|
79
|
+
parent_pack_count = pack_groups.count(&:has_parent?)
|
80
|
+
all_cross_pack_group_violations = pack_groups.flat_map(&:cross_group_violations)
|
81
|
+
|
82
|
+
all_metrics << GaugeMetric.for('all_pack_groups.count', all_pack_groups_count, package_tags)
|
83
|
+
all_metrics << GaugeMetric.for('child_packs.count', child_pack_count, package_tags)
|
84
|
+
all_metrics << GaugeMetric.for('parent_packs.count', parent_pack_count, package_tags)
|
85
|
+
all_metrics << GaugeMetric.for('all_pack_groups.privacy_violations.count', Metrics.file_count(all_cross_pack_group_violations.select(&:privacy?)), package_tags)
|
86
|
+
all_metrics << GaugeMetric.for('all_pack_groups.dependency_violations.count', Metrics.file_count(all_cross_pack_group_violations.select(&:dependency?)), package_tags)\
|
87
|
+
|
88
|
+
packs_by_group = {}
|
89
|
+
pack_groups.each do |pack_group|
|
90
|
+
pack_group.members.each do |member|
|
91
|
+
packs_by_group[member.name] = pack_group.name
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
inbound_violations_by_pack_group = {}
|
96
|
+
all_cross_pack_group_violations.group_by(&:to_package_name).each do |to_package_name, violations|
|
97
|
+
violations.each do |violation|
|
98
|
+
pack_group_for_violation = packs_by_group[violation.to_package_name]
|
99
|
+
inbound_violations_by_pack_group[pack_group_for_violation] ||= []
|
100
|
+
inbound_violations_by_pack_group[pack_group_for_violation] << violation
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
pack_groups.each do |pack_group|
|
105
|
+
tags = [
|
106
|
+
*package_tags,
|
107
|
+
Tag.for('pack_group', Metrics.humanized_package_name(pack_group.name)),
|
108
|
+
]
|
109
|
+
|
110
|
+
outbound_dependency_violations = pack_group.cross_group_violations.select(&:dependency?)
|
111
|
+
inbound_privacy_violations = inbound_violations_by_pack_group.fetch(pack_group.name, []).select(&:privacy?)
|
112
|
+
all_metrics << GaugeMetric.for('by_pack_group.outbound_dependency_violations.count', Metrics.file_count(outbound_dependency_violations), tags)
|
113
|
+
all_metrics << GaugeMetric.for('by_pack_group.inbound_privacy_violations.count', Metrics.file_count(inbound_privacy_violations), tags)
|
114
|
+
end
|
115
|
+
|
116
|
+
pack_groups.each do |from_pack_group|
|
117
|
+
violations_by_to_pack_group = from_pack_group.cross_group_violations.group_by do |violation|
|
118
|
+
packs_by_group[violation.to_package_name]
|
119
|
+
end
|
120
|
+
violations_by_to_pack_group.each do |to_pack_group_name, violations|
|
121
|
+
tags = [
|
122
|
+
*package_tags,
|
123
|
+
Tag.for('pack_group', Metrics.humanized_package_name(from_pack_group.name)),
|
124
|
+
Tag.for('to_pack_group', Metrics.humanized_package_name(to_pack_group_name)),
|
125
|
+
]
|
126
|
+
|
127
|
+
all_metrics << GaugeMetric.for('by_pack_group.outbound_dependency_violations.per_pack_group.count', Metrics.file_count(violations.select(&:dependency?)), tags)
|
128
|
+
all_metrics << GaugeMetric.for('by_pack_group.outbound_privacy_violations.per_pack_group.count', Metrics.file_count(violations.select(&:privacy?)), tags)
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
all_metrics
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|