xmi 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop_todo.yml +6 -7
- data/CHANGELOG.md +61 -0
- data/CLAUDE.md +2 -2
- data/lib/tasks/performance.rake +6 -8
- data/lib/xmi/performance/comparator.rb +81 -0
- data/lib/xmi/performance/helpers.rb +190 -0
- data/lib/xmi/performance/runner.rb +203 -0
- data/lib/xmi/performance/term.rb +58 -0
- data/lib/xmi/performance.rb +27 -0
- data/lib/xmi/sparx/index.rb +1 -0
- data/lib/xmi/uml/association_class.rb +1 -1
- data/lib/xmi/uml/component.rb +1 -1
- data/lib/xmi/uml/dependency.rb +1 -1
- data/lib/xmi/uml/extension.rb +1 -1
- data/lib/xmi/uml/owned_attribute.rb +3 -2
- data/lib/xmi/uml/owned_comment.rb +3 -2
- data/lib/xmi/uml/owned_element.rb +2 -2
- data/lib/xmi/uml/owned_end.rb +3 -2
- data/lib/xmi/uml/owned_parameter.rb +20 -2
- data/lib/xmi/uml/packaged_element.rb +21 -9
- data/lib/xmi/uml/signal.rb +1 -1
- data/lib/xmi/uml/stereotype.rb +1 -1
- data/lib/xmi/uml/uml_class.rb +1 -1
- data/lib/xmi/uml/usage.rb +1 -1
- data/lib/xmi/uml/value_specification.rb +10 -4
- data/lib/xmi/version.rb +1 -1
- data/lib/xmi.rb +1 -0
- metadata +7 -5
- data/lib/tasks/benchmark_runner.rb +0 -274
- data/lib/tasks/performance_comparator.rb +0 -88
- data/lib/tasks/performance_helpers.rb +0 -238
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 3a7a71b5331da8d8c45859ce5c634bd65bd2564ce76505bdeea512df0ffb5001
|
|
4
|
+
data.tar.gz: bceea821dc3bfc67feff1e43b4124288c02e1b0bf99351e32679e87f67594da3
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 0a67bce4eef99e51ab0213309b2d24ddaf0170c01a1c5c3094f2ba3ae2436cf9c3052a828545516650bd321f2445081fd310233c8ddaa082ca0e8d653b299029
|
|
7
|
+
data.tar.gz: 7d7b2f66f23144d1c27809e2d471f430e98794681da8944305ee701b889ee612c486cc9a8a118c34ef3c6fef1091f89a08007e9139215fd8fb9b88378411996c
|
data/.rubocop_todo.yml
CHANGED
|
@@ -113,9 +113,9 @@ Lint/ConstantDefinitionInBlock:
|
|
|
113
113
|
# Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes, Max.
|
|
114
114
|
Metrics/AbcSize:
|
|
115
115
|
Exclude:
|
|
116
|
-
- 'lib/
|
|
117
|
-
- 'lib/
|
|
118
|
-
- 'lib/
|
|
116
|
+
- 'lib/xmi/performance/runner.rb'
|
|
117
|
+
- 'lib/xmi/performance/comparator.rb'
|
|
118
|
+
- 'lib/xmi/performance/helpers.rb'
|
|
119
119
|
- 'lib/xmi/ea_root/code_generation.rb'
|
|
120
120
|
- 'lib/xmi/sparx/index.rb'
|
|
121
121
|
- 'lib/xmi/version_registry.rb'
|
|
@@ -131,7 +131,7 @@ Metrics/BlockLength:
|
|
|
131
131
|
# Configuration parameters: AllowedMethods, AllowedPatterns, Max.
|
|
132
132
|
Metrics/CyclomaticComplexity:
|
|
133
133
|
Exclude:
|
|
134
|
-
- 'lib/
|
|
134
|
+
- 'lib/xmi/performance/helpers.rb'
|
|
135
135
|
- 'lib/xmi/ea_root/code_generation.rb'
|
|
136
136
|
- 'lib/xmi/sparx/index.rb'
|
|
137
137
|
|
|
@@ -144,7 +144,7 @@ Metrics/MethodLength:
|
|
|
144
144
|
# Configuration parameters: AllowedMethods, AllowedPatterns, Max.
|
|
145
145
|
Metrics/PerceivedComplexity:
|
|
146
146
|
Exclude:
|
|
147
|
-
- 'lib/
|
|
147
|
+
- 'lib/xmi/performance/helpers.rb'
|
|
148
148
|
- 'lib/xmi/ea_root/code_generation.rb'
|
|
149
149
|
- 'lib/xmi/sparx/index.rb'
|
|
150
150
|
- 'lib/xmi/version_registry.rb'
|
|
@@ -155,7 +155,7 @@ Metrics/PerceivedComplexity:
|
|
|
155
155
|
# AllowedIdentifiers: TLS1_1, TLS1_2, capture3, iso8601, rfc1123_date, rfc822, rfc2822, rfc3339, x86_64
|
|
156
156
|
Naming/VariableNumber:
|
|
157
157
|
Exclude:
|
|
158
|
-
- 'lib/
|
|
158
|
+
- 'lib/xmi/performance/runner.rb'
|
|
159
159
|
- 'lib/xmi/v20110701.rb'
|
|
160
160
|
- 'lib/xmi/v20131001.rb'
|
|
161
161
|
- 'lib/xmi/v20161101.rb'
|
|
@@ -247,6 +247,5 @@ Style/BlockDelimiters:
|
|
|
247
247
|
# Configuration parameters: AllowedClasses.
|
|
248
248
|
Style/OneClassPerFile:
|
|
249
249
|
Exclude:
|
|
250
|
-
- 'lib/tasks/benchmark_runner.rb'
|
|
251
250
|
- 'lib/xmi.rb'
|
|
252
251
|
- 'lib/xmi/namespace/dynamic.rb'
|
data/CHANGELOG.md
CHANGED
|
@@ -9,6 +9,67 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
9
9
|
|
|
10
10
|
### Added
|
|
11
11
|
|
|
12
|
+
- `PackagedElement#nested_classifier` with `<nestedClassifier>` mapping
|
|
13
|
+
(Sparx EA nesting for classifiers owned by classes), covered by
|
|
14
|
+
`Xmi::Sparx::Index` lookups
|
|
15
|
+
|
|
16
|
+
### Changed
|
|
17
|
+
|
|
18
|
+
- `OwnedAttribute`/`OwnedEnd`/`OwnedParameter` serialize `lowerValue`
|
|
19
|
+
before `upperValue`, matching Sparx EA's element order. Serialization
|
|
20
|
+
only — parsing accepts either order and is unaffected
|
|
21
|
+
|
|
22
|
+
### Documented
|
|
23
|
+
|
|
24
|
+
- `OwnedParameter#type` keeps its xmi-namespaced mapping, and the
|
|
25
|
+
limitation is now stated in the code and pinned by specs.
|
|
26
|
+
`xmi:type` is the XMI metaclass discriminator and `type` is Sparx's
|
|
27
|
+
classifier reference — two different attributes. lutaml-model matches
|
|
28
|
+
by local name, so it collapses them into one slot and only the last
|
|
29
|
+
one read survives re-serialization. Keeping the slot namespaced
|
|
30
|
+
preserves the discriminator; restoring Sparx's classifier reference is
|
|
31
|
+
the Sparx exporter's job. Modelling both needs namespace-disjoint
|
|
32
|
+
attribute deserialization upstream (lutaml-model#744).
|
|
33
|
+
|
|
34
|
+
## [0.6.2] - 2026-07-18
|
|
35
|
+
|
|
36
|
+
### Added
|
|
37
|
+
|
|
38
|
+
- `Xmi::Performance` namespace for benchmark tooling (`Helpers`, `Comparator`, `Runner`) with autoload
|
|
39
|
+
- Polymorphic dispatch contract specs asserting frozen + default-value behavior at the data level
|
|
40
|
+
- Fallback coverage for `OwnedAttribute#upper_value`, `OwnedEnd#upper_value`, `OwnedParameter#upper_value` (previously only `Slot#value` was tested)
|
|
41
|
+
- Round-trip coverage for `uml:Component` and unknown `xmi:type` discriminators
|
|
42
|
+
- `spec/xmi/performance_spec.rb` covering autoload, class cloning, and namespace independence
|
|
43
|
+
|
|
44
|
+
### Fixed
|
|
45
|
+
|
|
46
|
+
- Deep-freeze the inner `class_map` hash on both `PACKAGED_ELEMENT` and `VALUE_SPECIFICATION` polymorphic maps. The outer `.freeze` was shallow — the inner hash was mutable at runtime
|
|
47
|
+
- `cached_fixture` ivar caching bug: the cache lived on per-example `self` and was recreated empty on every call, defeating its purpose. Replaced with a top-level `FIXTURE_CONTENT_CACHE` constant
|
|
48
|
+
|
|
49
|
+
### Changed
|
|
50
|
+
|
|
51
|
+
- Performance task constants moved into `Xmi::Performance` namespace: `PerformanceComparator` → `Xmi::Performance::Comparator`, `PerformanceHelpers` → `Xmi::Performance::Helpers`, `BenchmarkRunner` → `Xmi::Performance::Runner`. Dual-load mechanism replaced with class cloning via `const_set`
|
|
52
|
+
- Data-drive dispatch and subclass-schema specs from the class_map so adding a subclass + map entry auto-gains coverage (OCP for the test layer)
|
|
53
|
+
- Replace `.send(...)` with `.public_send(...)` in sparx extension specs for safer dynamic dispatch
|
|
54
|
+
- Normalize `require_relative` to `require "xmi"` in 8 spec files for consistency with the rest of the suite
|
|
55
|
+
- Correct polymorphic map comment terminology (`default_proc` → `default value`) and document the two-path fallback
|
|
56
|
+
|
|
57
|
+
### Removed
|
|
58
|
+
|
|
59
|
+
- `lib/tasks/performance_helpers.rb`, `lib/tasks/performance_comparator.rb`, `lib/tasks/benchmark_runner.rb` (moved to `lib/xmi/performance/`)
|
|
60
|
+
- Top-level constants `PerformanceComparator`, `PerformanceHelpers`, `BenchmarkRunner`, `Term` (moved into `Xmi::Performance`)
|
|
61
|
+
- Stale `TODO.next/01` and `TODO 02` references from comments (scratchpads were deleted; Phase A explanations kept)
|
|
62
|
+
|
|
63
|
+
## [0.6.1] - 2026-07-09
|
|
64
|
+
|
|
65
|
+
### Added
|
|
66
|
+
|
|
67
|
+
- `Xmi::Uml::Component` subclass for `<packagedElement xmi:type="uml:Component">` (UML 2.5 §12.3)
|
|
68
|
+
- `uml:Component` entry in `PACKAGED_ELEMENT_POLYMORPHIC_MAP`
|
|
69
|
+
- Polymorphic dispatch fallback: unknown or missing `xmi:type` resolves to the abstract base class via `Hash` default value, avoiding the lutaml-model `Object.const_get(nil)` `TypeError` on real Sparx XMI containing types not yet modelled
|
|
70
|
+
|
|
71
|
+
### Added
|
|
72
|
+
|
|
12
73
|
- Namespace-bound registers for version-aware type resolution
|
|
13
74
|
- `Register#bind_namespace` for binding registers to namespaces
|
|
14
75
|
- `Register#resolve_in_namespace` for namespace-aware type lookup
|
data/CLAUDE.md
CHANGED
|
@@ -150,7 +150,7 @@ Two attributes use lutaml-model's polymorphic dispatch on `xmi:type`:
|
|
|
150
150
|
- `PackagedElement.packaged_element` (and `UmlModel.packaged_element`) — dispatches to typed subclasses (`UmlClass`, `Association`, `Interface`, `InstanceSpecification`, etc.). Map: `Xmi::Uml::PACKAGED_ELEMENT_POLYMORPHIC_MAP` in `lib/xmi/uml/packaged_element.rb`.
|
|
151
151
|
- `Slot.value`, `OwnedAttribute.upper_value`/`lower_value`/`default_value`, `OwnedEnd.upper_value`/`lower_value`/`default_value`, `OwnedParameter.upper_value`/`lower_value`/`default_value` — dispatch to ValueSpecification subclasses (`OpaqueExpression`, `LiteralString`, `LiteralInteger`, etc.). Map: `Xmi::Uml::VALUE_SPECIFICATION_POLYMORPHIC_MAP` in `lib/xmi/uml/value_specification.rb`.
|
|
152
152
|
|
|
153
|
-
**
|
|
153
|
+
**Fallback contract:** unknown or missing `xmi:type` resolves to the abstract base (`PackagedElement` / `ValueSpecification`) via the class_map's Hash default value — no crash. Locked in by `spec/xmi/uml/polymorphic_map_contract_spec.rb` and `spec/xmi/uml/polymorphic_robustness_spec.rb`. Two paths land at the base: a missing discriminator short-circuits upstream (the declared attribute type is used); an unknown discriminator hits the Hash default. Residual upstream gap — modelling `xmi:type` and unprefixed `type` as separate slots on one element — is tracked in lutaml-model#744.
|
|
154
154
|
|
|
155
155
|
### Key Files
|
|
156
156
|
|
|
@@ -188,7 +188,7 @@ A shared constant is available at `Xmi::Sparx::VALUE_MAP`.
|
|
|
188
188
|
|
|
189
189
|
## Known Issues
|
|
190
190
|
|
|
191
|
-
|
|
191
|
+
Attribute deserialization in lutaml-model is not namespace-disjoint: same-local-name attributes in different namespaces collapse into one model slot, document-order dependent. This affects modelling `xmi:type` (the XMI discriminator) and unprefixed `type` (Sparx's classifier reference) as separate attributes on one element. Tracked in lutaml-model#744 with a repro. Serialization is not affected — it is already namespace-disjoint.
|
|
192
192
|
|
|
193
193
|
## Limitations
|
|
194
194
|
|
data/lib/tasks/performance.rake
CHANGED
|
@@ -1,34 +1,32 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
require_relative "benchmark_runner"
|
|
3
|
+
require "xmi"
|
|
5
4
|
|
|
6
5
|
desc "Run performance benchmarks"
|
|
7
6
|
namespace :performance do
|
|
8
7
|
desc "Compare performance of current branch against base branch (default: main)"
|
|
9
8
|
task :compare do
|
|
10
|
-
|
|
9
|
+
Xmi::Performance::Comparator.new.run
|
|
11
10
|
end
|
|
12
11
|
|
|
13
12
|
desc "Run benchmarks on current branch only (for development)"
|
|
14
13
|
task :run do
|
|
15
|
-
runner =
|
|
14
|
+
runner = Xmi::Performance::Runner.new(run_time: 5)
|
|
16
15
|
runner.run_benchmarks
|
|
17
16
|
end
|
|
18
17
|
|
|
19
18
|
desc "Quick benchmark run (faster, less accurate)"
|
|
20
19
|
task :quick do
|
|
21
|
-
runner =
|
|
20
|
+
runner = Xmi::Performance::Runner.new(run_time: 2, warmup: 1)
|
|
22
21
|
runner.run_benchmarks
|
|
23
22
|
end
|
|
24
23
|
|
|
25
24
|
desc "Run benchmarks and output as JSON"
|
|
26
25
|
task :json do
|
|
27
26
|
require "json"
|
|
28
|
-
runner =
|
|
27
|
+
runner = Xmi::Performance::Runner.new(run_time: 5)
|
|
29
28
|
|
|
30
|
-
|
|
31
|
-
results = runner.send(:run_benchmarks)
|
|
29
|
+
results = runner.run_benchmarks
|
|
32
30
|
|
|
33
31
|
output = results.each_with_object({}) do |(label, metrics), h|
|
|
34
32
|
ips = (metrics[:lower] + metrics[:upper]) / 2.0
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
|
|
5
|
+
module Xmi
|
|
6
|
+
module Performance
|
|
7
|
+
# Orchestrates the performance comparison flow: loads the Runner
|
|
8
|
+
# into Base and Current namespaces, runs both, and reports
|
|
9
|
+
# regressions.
|
|
10
|
+
class Comparator
|
|
11
|
+
REPO_ROOT = File.expand_path(File.join(__dir__, "..", "..", ".."))
|
|
12
|
+
DEFAULT_RUN_TIME = 10
|
|
13
|
+
DEFAULT_THRESHOLD = 0.10 # 10% (more lenient for complex operations)
|
|
14
|
+
DEFAULT_BASE = "main"
|
|
15
|
+
|
|
16
|
+
def run
|
|
17
|
+
setup_environment
|
|
18
|
+
run_benchmarks_comparison
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
private
|
|
22
|
+
|
|
23
|
+
def setup_environment
|
|
24
|
+
Dir.chdir(REPO_ROOT)
|
|
25
|
+
Helpers.load_runner_into(Helpers::Current)
|
|
26
|
+
Helpers.load_runner_into(Helpers::Base)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def run_benchmarks_comparison
|
|
30
|
+
all_current = {}
|
|
31
|
+
all_base = {}
|
|
32
|
+
|
|
33
|
+
puts Term.header("Performance Comparison", color: CYAN)
|
|
34
|
+
puts
|
|
35
|
+
puts " #{DIM}Comparing#{CLEAR}:"
|
|
36
|
+
puts " #{CYAN} Current#{CLEAR}: #{Helpers.current_branch}"
|
|
37
|
+
puts " #{CYAN} Base#{CLEAR}: #{DEFAULT_BASE}"
|
|
38
|
+
puts " #{CYAN} Threshold#{CLEAR}: #{(DEFAULT_THRESHOLD * 100).round(0)}% regression allowed"
|
|
39
|
+
puts
|
|
40
|
+
|
|
41
|
+
base_runner = Helpers::Base::Runner.new(
|
|
42
|
+
run_time: DEFAULT_RUN_TIME,
|
|
43
|
+
)
|
|
44
|
+
current_runner = Helpers::Current::Runner.new(
|
|
45
|
+
run_time: DEFAULT_RUN_TIME,
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
Helpers.run_benchmarks(
|
|
49
|
+
base_runner,
|
|
50
|
+
current_runner,
|
|
51
|
+
DEFAULT_THRESHOLD,
|
|
52
|
+
all_base,
|
|
53
|
+
all_current,
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
summary = Helpers.summary_report(
|
|
57
|
+
all_current,
|
|
58
|
+
all_base,
|
|
59
|
+
DEFAULT_BASE,
|
|
60
|
+
DEFAULT_RUN_TIME,
|
|
61
|
+
DEFAULT_THRESHOLD,
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
handle_results(summary)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def handle_results(summary)
|
|
68
|
+
puts
|
|
69
|
+
if summary[:regressions].any?
|
|
70
|
+
puts " #{RED}#{BOLD}❌ PERFORMANCE REGRESSIONS DETECTED#{CLEAR}"
|
|
71
|
+
puts " #{RED}#{summary[:regressions].length} benchmark(s) regressed beyond threshold#{CLEAR}"
|
|
72
|
+
puts
|
|
73
|
+
exit(1)
|
|
74
|
+
else
|
|
75
|
+
puts " #{GREEN}#{BOLD}✅ ALL BENCHMARKS PASSED#{CLEAR}"
|
|
76
|
+
puts
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "open3"
|
|
5
|
+
require "tmpdir"
|
|
6
|
+
require "fileutils"
|
|
7
|
+
|
|
8
|
+
module Xmi
|
|
9
|
+
module Performance
|
|
10
|
+
# Shared helpers for the performance benchmark rake tasks.
|
|
11
|
+
# Provides git/process helpers and the dual-namespace Runner loader
|
|
12
|
+
# used by the comparison flow. Terminal formatting lives in
|
|
13
|
+
# Xmi::Performance::Term; color codes on Xmi::Performance.
|
|
14
|
+
module Helpers
|
|
15
|
+
# Dual-namespace containers. The Runner class is cloned into
|
|
16
|
+
# each so that Base and Current hold independent class objects
|
|
17
|
+
# (preserving the shape of the comparison API even though both
|
|
18
|
+
# currently resolve to the same benchmark code).
|
|
19
|
+
module Base
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
module Current
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
class << self
|
|
26
|
+
# Clone Xmi::Performance::Runner into the given namespace as
|
|
27
|
+
# `::Runner`. Replaces the old module_eval-of-a-top-level-file
|
|
28
|
+
# mechanism, which required require_relative and a top-level
|
|
29
|
+
# class definition incompatible with autoload.
|
|
30
|
+
def load_runner_into(namespace)
|
|
31
|
+
Xmi::Performance::Runner # force autoload
|
|
32
|
+
namespace.const_set(:Runner, Xmi::Performance::Runner.clone)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def ruby_exec(cmd, env: {})
|
|
36
|
+
Open3.capture3(env, cmd)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def current_branch
|
|
40
|
+
stdout, = ruby_exec("git rev-parse --abbrev-ref HEAD")
|
|
41
|
+
stdout.strip
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def run_benchmarks(base_runner, current_runner, threshold, all_base,
|
|
45
|
+
all_current)
|
|
46
|
+
base_results = base_runner.run_benchmarks
|
|
47
|
+
curr_results = current_runner.run_benchmarks
|
|
48
|
+
|
|
49
|
+
all_base.merge!(base_results)
|
|
50
|
+
all_current.merge!(curr_results)
|
|
51
|
+
|
|
52
|
+
comparison_rows = []
|
|
53
|
+
|
|
54
|
+
curr_results.each do |label, result|
|
|
55
|
+
base_result = base_results[label]
|
|
56
|
+
cmp = compare_metrics(label, result, base_result, threshold)
|
|
57
|
+
comparison_rows << cmp
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
print_comparison_table(comparison_rows, threshold)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def print_comparison_table(comparison_rows, threshold)
|
|
64
|
+
rows = comparison_rows.map do |cmp|
|
|
65
|
+
{
|
|
66
|
+
benchmark: cmp[:label],
|
|
67
|
+
base_ips: cmp[:base_ips]&.round(1),
|
|
68
|
+
curr_ips: cmp[:curr_ips]&.round(1),
|
|
69
|
+
change: cmp[:change] ? "#{(cmp[:change] * 100).round(1)}%" : "N/A",
|
|
70
|
+
status: if cmp[:base_ips].nil?
|
|
71
|
+
"NEW"
|
|
72
|
+
elsif cmp[:change] < -threshold
|
|
73
|
+
"REGRESSED"
|
|
74
|
+
else
|
|
75
|
+
"OK"
|
|
76
|
+
end,
|
|
77
|
+
}
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
return if rows.empty?
|
|
81
|
+
|
|
82
|
+
puts " #{'Benchmark'.ljust(40)} #{'Base IPS'.rjust(12)} #{'Curr IPS'.rjust(12)} #{'Change'.rjust(10)} #{'Status'.rjust(10)}"
|
|
83
|
+
puts " #{DIM}#{'─' * 86}#{CLEAR}"
|
|
84
|
+
|
|
85
|
+
rows.each do |row|
|
|
86
|
+
status_color = case row[:status]
|
|
87
|
+
when "REGRESSED" then RED
|
|
88
|
+
when "NEW" then YELLOW
|
|
89
|
+
else GREEN
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
puts " #{row[:benchmark].ljust(40)} #{format('%-12.1f',
|
|
93
|
+
row[:base_ips] || 0)} #{format('%-12.1f',
|
|
94
|
+
row[:curr_ips] || 0)} #{format('%-10s', row[:change]).gsub('%',
|
|
95
|
+
'%%')} #{status_color}#{row[:status].rjust(10)}#{CLEAR}"
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
puts
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def compare_metrics(label, curr, base, threshold)
|
|
102
|
+
unless base
|
|
103
|
+
return { label: label, base_ips: nil, curr_ips: nil, change: nil,
|
|
104
|
+
regressed: false }
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
base_ips = base.fetch(:lower)
|
|
108
|
+
curr_ips = curr.fetch(:upper)
|
|
109
|
+
change = (curr_ips - base_ips) / base_ips.to_f
|
|
110
|
+
|
|
111
|
+
{
|
|
112
|
+
label: label,
|
|
113
|
+
base_ips: base_ips,
|
|
114
|
+
curr_ips: curr_ips,
|
|
115
|
+
change: change,
|
|
116
|
+
regressed: change < -threshold,
|
|
117
|
+
}
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def summary_report(current_results, base_results, base, run_time, threshold)
|
|
121
|
+
summary = {
|
|
122
|
+
run_time: run_time,
|
|
123
|
+
threshold: threshold,
|
|
124
|
+
branch: current_branch,
|
|
125
|
+
base: base,
|
|
126
|
+
regressions: [],
|
|
127
|
+
new_benchmarks: [],
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
current_results.each do |label, metrics|
|
|
131
|
+
base_result = base_results[label]
|
|
132
|
+
cmp = compare_metrics(label, metrics, base_result, threshold)
|
|
133
|
+
|
|
134
|
+
if base_result.nil?
|
|
135
|
+
summary[:new_benchmarks] << label
|
|
136
|
+
next
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
next unless cmp[:regressed]
|
|
140
|
+
|
|
141
|
+
summary[:regressions] << {
|
|
142
|
+
label: label,
|
|
143
|
+
base_ips: cmp[:base_ips],
|
|
144
|
+
curr_ips: cmp[:curr_ips],
|
|
145
|
+
delta_fraction: cmp[:change],
|
|
146
|
+
}
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
log_regressions(summary[:regressions], threshold)
|
|
150
|
+
log_new_benchmarks(summary[:new_benchmarks])
|
|
151
|
+
summary
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def log_new_benchmarks(new_benchmarks)
|
|
155
|
+
return if new_benchmarks.empty?
|
|
156
|
+
|
|
157
|
+
puts
|
|
158
|
+
puts "#{YELLOW}🆕 New benchmarks (not in base branch):#{CLEAR}"
|
|
159
|
+
new_benchmarks.each do |label|
|
|
160
|
+
puts " • #{label}"
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
def log_regressions(regressions, threshold)
|
|
165
|
+
return if regressions.empty?
|
|
166
|
+
|
|
167
|
+
puts
|
|
168
|
+
puts "#{RED}⚠️ Performance Regressions Detected#{CLEAR}"
|
|
169
|
+
puts "#{RED} (< -#{(threshold * 100).round(2)}% IPS)#{CLEAR}"
|
|
170
|
+
puts
|
|
171
|
+
regressions.each do |regression|
|
|
172
|
+
delta = regression[:delta_fraction]
|
|
173
|
+
base_ips = regression[:base_ips]
|
|
174
|
+
curr_ips = regression[:curr_ips]
|
|
175
|
+
|
|
176
|
+
delta_str = delta ? format("%+0.2f%%", delta * 100) : "N/A"
|
|
177
|
+
base_str = base_ips ? format("%.2f", base_ips) : "N/A"
|
|
178
|
+
curr_str = curr_ips ? format("%.2f", curr_ips) : "N/A"
|
|
179
|
+
|
|
180
|
+
puts " #{BOLD}#{regression[:label]}#{CLEAR}"
|
|
181
|
+
puts " #{GRAY}base: #{base_str} IPS#{CLEAR}"
|
|
182
|
+
puts " #{RED}curr: #{curr_str} IPS#{CLEAR}"
|
|
183
|
+
puts " #{RED}change: #{delta_str}#{CLEAR}"
|
|
184
|
+
puts
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
end
|
|
189
|
+
end
|
|
190
|
+
end
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "benchmark/ips"
|
|
4
|
+
require "stringio"
|
|
5
|
+
|
|
6
|
+
module Xmi
|
|
7
|
+
module Performance
|
|
8
|
+
# Runs XMI parsing benchmarks against fixture files and reports
|
|
9
|
+
# iterations-per-second with error margins.
|
|
10
|
+
class Runner
|
|
11
|
+
REPO_ROOT = File.expand_path(File.join(__dir__, "..", "..", ".."))
|
|
12
|
+
|
|
13
|
+
DEFAULT_RUN_TIME = 5
|
|
14
|
+
DEFAULT_WARMUP = 2
|
|
15
|
+
|
|
16
|
+
CATEGORIES = {
|
|
17
|
+
xmi_parsing: {
|
|
18
|
+
name: "XMI Parsing",
|
|
19
|
+
icon: "📄",
|
|
20
|
+
description: "XMI parsing performance tests. Measures how quickly we can convert XMI files into Ruby objects.",
|
|
21
|
+
failure_means: "Slow XMI parsing impacts all downstream operations. A regression here means users will experience delays when processing XMI documents.",
|
|
22
|
+
compare_against: "Previous branch (main).",
|
|
23
|
+
},
|
|
24
|
+
}.freeze
|
|
25
|
+
|
|
26
|
+
BENCHMARKS = {
|
|
27
|
+
xmi_parsing: [
|
|
28
|
+
{ name: "XMI 2.4.2 (small)", method: :xmi_parse_242_small,
|
|
29
|
+
desc: "XMI 2.4.2 ~100KB file" },
|
|
30
|
+
{ name: "XMI 2.4.2 (medium)", method: :xmi_parse_242_medium,
|
|
31
|
+
desc: "XMI 2.4.2 ~500KB file with extensions" },
|
|
32
|
+
{ name: "XMI 2.4.2 (large)", method: :xmi_parse_242_large,
|
|
33
|
+
desc: "XMI 2.4.2 ~3.5MB file" },
|
|
34
|
+
{ name: "XMI 2.5.1", method: :xmi_parse_251,
|
|
35
|
+
desc: "XMI 2.5.1 ~100KB file" },
|
|
36
|
+
],
|
|
37
|
+
}.freeze
|
|
38
|
+
|
|
39
|
+
FIXTURES = {
|
|
40
|
+
xmi_parse_242_small: "spec/fixtures/xmi-v2-4-2-default.xmi",
|
|
41
|
+
xmi_parse_242_medium: "spec/fixtures/xmi-v2-4-2-default-with-citygml.xmi",
|
|
42
|
+
xmi_parse_242_large: "spec/fixtures/full-242.xmi",
|
|
43
|
+
xmi_parse_251: "spec/fixtures/ea-xmi-2.5.1.xmi",
|
|
44
|
+
}.freeze
|
|
45
|
+
|
|
46
|
+
def initialize(run_time: nil, warmup: nil, benchmark: nil)
|
|
47
|
+
@run_time = run_time || DEFAULT_RUN_TIME
|
|
48
|
+
@warmup = warmup || DEFAULT_WARMUP
|
|
49
|
+
@benchmark = benchmark
|
|
50
|
+
@results = {}
|
|
51
|
+
@env_shown = false
|
|
52
|
+
@all_results = []
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def run_benchmarks
|
|
56
|
+
Term.header("XMI Performance Benchmarks", color: CYAN)
|
|
57
|
+
|
|
58
|
+
unless @env_shown
|
|
59
|
+
Term.env_info(RUBY_VERSION, RUBY_PLATFORM)
|
|
60
|
+
@env_shown = true
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
BENCHMARKS.each do |category, tests|
|
|
64
|
+
run_category(category, tests)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
print_summary
|
|
68
|
+
|
|
69
|
+
@results
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
private
|
|
73
|
+
|
|
74
|
+
def run_category(category, tests)
|
|
75
|
+
config = CATEGORIES[category]
|
|
76
|
+
Term.category(
|
|
77
|
+
config[:name],
|
|
78
|
+
icon: config[:icon],
|
|
79
|
+
description: config[:description],
|
|
80
|
+
failure_means: config[:failure_means],
|
|
81
|
+
compare_against: config[:compare_against],
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
category_results = []
|
|
85
|
+
|
|
86
|
+
tests.each do |test|
|
|
87
|
+
original_stdout = $stdout
|
|
88
|
+
$stdout = StringIO.new
|
|
89
|
+
|
|
90
|
+
result = run_single_test(test[:method])
|
|
91
|
+
(result[:lower] + result[:upper]) / 2.0
|
|
92
|
+
category_results << { name: test[:name], result: result }
|
|
93
|
+
|
|
94
|
+
$stdout = original_stdout
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
puts " #{'Benchmark'.ljust(40)} #{'IPS'.rjust(12)} #{'Deviation'.rjust(12)}"
|
|
98
|
+
puts " #{DIM}#{Term::HL * 66}#{CLEAR}"
|
|
99
|
+
|
|
100
|
+
category_results.each do |r|
|
|
101
|
+
ips = (r[:result][:lower] + r[:result][:upper]) / 2.0
|
|
102
|
+
deviation = calculate_deviation(r[:result])
|
|
103
|
+
label = "#{config[:name]}: #{r[:name]}"
|
|
104
|
+
@all_results << { label: label, ips: ips }
|
|
105
|
+
@results[label] = r[:result]
|
|
106
|
+
|
|
107
|
+
puts " #{r[:name].ljust(40)} #{format('%.2f',
|
|
108
|
+
ips).rjust(12)} #{format('%.1f%%',
|
|
109
|
+
deviation).rjust(12)}"
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
puts
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def run_single_test(method)
|
|
116
|
+
fixture_path = FIXTURES[method]
|
|
117
|
+
raise "Unknown fixture: #{method}" unless fixture_path
|
|
118
|
+
|
|
119
|
+
full_path = File.join(REPO_ROOT, fixture_path)
|
|
120
|
+
full_path = fixture_path unless File.exist?(full_path)
|
|
121
|
+
|
|
122
|
+
xml_content = File.read(full_path)
|
|
123
|
+
|
|
124
|
+
case method
|
|
125
|
+
when :xmi_parse_242_small, :xmi_parse_242_medium, :xmi_parse_242_large, :xmi_parse_251
|
|
126
|
+
measure_time { Xmi::Sparx::Root.parse_xml(xml_content) }
|
|
127
|
+
else
|
|
128
|
+
raise "Unknown benchmark: #{method}"
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def measure(&)
|
|
133
|
+
job = Benchmark::IPS::Job.new
|
|
134
|
+
job.config(time: @run_time, warmup: @warmup)
|
|
135
|
+
job.report("test", &)
|
|
136
|
+
job.run
|
|
137
|
+
|
|
138
|
+
entry = job.full_report.entries.first
|
|
139
|
+
samples = entry.stats.samples
|
|
140
|
+
|
|
141
|
+
return { lower: 0, upper: 0 } if samples.empty?
|
|
142
|
+
|
|
143
|
+
mean = samples.sum.to_f / samples.size
|
|
144
|
+
variance = samples.sum { |x| (x - mean)**2 } / (samples.size - 1)
|
|
145
|
+
std_dev = Math.sqrt(variance)
|
|
146
|
+
error_margin = std_dev / mean
|
|
147
|
+
error_pct = error_margin.round(4)
|
|
148
|
+
|
|
149
|
+
{ lower: mean.round(4) * (1 - error_pct),
|
|
150
|
+
upper: mean.round(4) * (1 + error_pct) }
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def measure_time
|
|
154
|
+
times = []
|
|
155
|
+
iterations = 5
|
|
156
|
+
|
|
157
|
+
iterations.times do
|
|
158
|
+
start_t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
159
|
+
yield
|
|
160
|
+
finish_t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
161
|
+
times << (finish_t - start_t)
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
mean = times.sum / times.size
|
|
165
|
+
variance = times.sum { |t| (t - mean)**2 } / (times.size - 1)
|
|
166
|
+
std_dev = Math.sqrt(variance)
|
|
167
|
+
|
|
168
|
+
lower_time = [mean - std_dev, mean * 0.5].max
|
|
169
|
+
lower_ips = (1.0 / (lower_time * 1.5)).round(4)
|
|
170
|
+
upper_ips = (1.0 / mean).round(4)
|
|
171
|
+
|
|
172
|
+
if mean < 0.001
|
|
173
|
+
upper_ips = (1.0 / mean).round(4)
|
|
174
|
+
lower_ips = (upper_ips * 0.8).round(4)
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
{ lower: lower_ips, upper: upper_ips }
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
def calculate_deviation(metrics)
|
|
181
|
+
return 0 if metrics[:upper].zero?
|
|
182
|
+
|
|
183
|
+
((metrics[:upper] - metrics[:lower]) / metrics[:upper] * 100).round(1)
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
def print_summary
|
|
187
|
+
puts
|
|
188
|
+
Term.sep(width: 78)
|
|
189
|
+
puts
|
|
190
|
+
puts " #{BOLD}#{MAGENTA}SUMMARY#{CLEAR}"
|
|
191
|
+
puts
|
|
192
|
+
|
|
193
|
+
@all_results.each do |r|
|
|
194
|
+
puts " #{r[:label].ljust(60)} #{format('%.2f', r[:ips]).rjust(10)} IPS"
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
puts
|
|
198
|
+
puts " #{DIM}#{@all_results.length} benchmarks completed#{CLEAR}"
|
|
199
|
+
puts
|
|
200
|
+
end
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
end
|