kensho-rspec 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +177 -0
- data/lib/kensho/_schema.rb +211 -0
- data/lib/kensho/rspec/formatter.rb +559 -0
- data/lib/kensho/rspec/helpers.rb +187 -0
- data/lib/kensho/rspec/state.rb +64 -0
- data/lib/kensho/rspec/version.rb +7 -0
- data/lib/kensho/rspec.rb +24 -0
- metadata +72 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: b97dc4db016aa21aa420a9b63740cabe981c268aae5327934f4ee2e3df61d7ab
|
|
4
|
+
data.tar.gz: 9a93a8174851bc749cb9a645ad50a981cb649773ca4e88735fd83f1b63e6e995
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 0143d4de5952dcaefa21537e0a7a99b473c67eacff4fd8d77490dbb4e8becbab56039df5461bd4b62cdfb4a6bc5f6b157cb450e5b763eb268e483acbb9e783be
|
|
7
|
+
data.tar.gz: 9af59332e7f1687b4368bd28d6873dc5dbbe3e3d41e5da2823ef548732f8ed0edfc8a0c5b33ea8ffae72f159158e9d952286b783d587b69aae659c82268092b3
|
data/README.md
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
# kensho-rspec
|
|
2
|
+
|
|
3
|
+
An RSpec 3+ formatter that emits the canonical [Kensho v1](../schema) JSON
|
|
4
|
+
format. Run your spec, point the `kensho` CLI at `kensho-results/`, and
|
|
5
|
+
get a self-contained static HTML report.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
```ruby
|
|
10
|
+
# Gemfile
|
|
11
|
+
gem 'kensho-rspec', require: false, group: :test
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
bundle install
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Run
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
# minimal — adds the formatter alongside whatever you already use
|
|
22
|
+
bundle exec rspec --format Kensho::RSpec::Formatter
|
|
23
|
+
|
|
24
|
+
# combine with the default progress formatter
|
|
25
|
+
bundle exec rspec --format documentation --format Kensho::RSpec::Formatter
|
|
26
|
+
|
|
27
|
+
# render the HTML report (uses the JS CLI from the same monorepo):
|
|
28
|
+
npx kensho generate
|
|
29
|
+
npx kensho open
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
You can also pin the formatter from `.rspec`:
|
|
33
|
+
|
|
34
|
+
```
|
|
35
|
+
--require kensho/rspec
|
|
36
|
+
--format Kensho::RSpec::Formatter
|
|
37
|
+
--format documentation
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## What it produces
|
|
41
|
+
|
|
42
|
+
- `kensho-results/run.json` — run manifest (project, env, totals, timing).
|
|
43
|
+
- `kensho-results/cases/<stableId>.json` — one file per example.
|
|
44
|
+
- `kensho-results/attachments/<caseId>/...` — files registered via `Kensho.attach`.
|
|
45
|
+
|
|
46
|
+
Each case gets a **stable id** (`tc_<16 hex>`) hashed from its full
|
|
47
|
+
description + file path so test history correlates across runs and
|
|
48
|
+
across adapters (the JS adapters use the same FNV-1a-based hash).
|
|
49
|
+
|
|
50
|
+
## Metadata we read
|
|
51
|
+
|
|
52
|
+
| Metadata | Effect |
|
|
53
|
+
| ---------------------------------------------- | --------------------------------------------------------- |
|
|
54
|
+
| `it 'foo', severity: :critical do` | Sets `case.severity`. Allowed: `blocker`, `critical`, `normal`, `minor`, `trivial`. |
|
|
55
|
+
| `it 'foo', :critical do` | Symbol shorthand. Same five severity names work. |
|
|
56
|
+
| `it 'foo', severity_blocker: true do` | Long-form alias. |
|
|
57
|
+
| `Kensho::Feature('Cart')` (inside `describe`) | Sets `case.behavior.feature`. |
|
|
58
|
+
| `Kensho::Epic('Checkout')` | Sets `case.behavior.epic`. |
|
|
59
|
+
| `Kensho::Story('Empty cart shows CTA')` | Sets `case.behavior.scenario`. |
|
|
60
|
+
| `it 'foo', owner: 'alice' do` | Sets `case.owner`. |
|
|
61
|
+
| `it 'foo', kensho_labels: { team: 'cart' } do` | Free-form `key=value` → `case.labels`. |
|
|
62
|
+
| `it 'foo', kensho_links: [{ kind: 'jira', url: '…', label: 'PROJ-123' }]` | Adds entries to `case.links`. |
|
|
63
|
+
| Any other symbol metadata (e.g. `:smoke`) | Becomes a tag on `case.tags`. |
|
|
64
|
+
| `it 'foo', tags: %w[smoke regression]` | Adds explicit tags. |
|
|
65
|
+
| `with_them` rows (rspec-parameterized) | Each row's variables → `case.parameters[]`. |
|
|
66
|
+
|
|
67
|
+
```ruby
|
|
68
|
+
require 'kensho/rspec'
|
|
69
|
+
|
|
70
|
+
RSpec.describe 'Cart' do
|
|
71
|
+
Kensho::Feature('Cart')
|
|
72
|
+
Kensho::Epic('Checkout')
|
|
73
|
+
|
|
74
|
+
it 'totals correctly', :critical, owner: 'alice',
|
|
75
|
+
kensho_labels: { team: 'growth' },
|
|
76
|
+
kensho_links: [{ kind: 'jira', url: 'https://jira.example.com/browse/PROJ-123', label: 'PROJ-123' }] do
|
|
77
|
+
expect(1 + 1).to eq(2)
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
## Helper API
|
|
83
|
+
|
|
84
|
+
```ruby
|
|
85
|
+
require 'kensho/rspec'
|
|
86
|
+
|
|
87
|
+
RSpec.describe 'Login' do
|
|
88
|
+
it 'submits credentials' do
|
|
89
|
+
Kensho.step('open the login page') do
|
|
90
|
+
visit '/login'
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
Kensho.step('submit credentials') do
|
|
94
|
+
fill_in 'user', with: 'demo'
|
|
95
|
+
click_button 'Sign in'
|
|
96
|
+
|
|
97
|
+
Kensho.step('verify redirect') do
|
|
98
|
+
expect(page.current_url).to end_with('/home')
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
Kensho.label('team', 'growth')
|
|
103
|
+
Kensho.link('https://jira.example.com/browse/PROJ-123',
|
|
104
|
+
kind: 'jira', label: 'PROJ-123')
|
|
105
|
+
|
|
106
|
+
page.save_screenshot('/tmp/login.png')
|
|
107
|
+
Kensho.attach('/tmp/login.png', kind: 'screenshot')
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
| Helper | What it does |
|
|
113
|
+
| ----------------------------------------------- | ------------------------------------------------------- |
|
|
114
|
+
| `Kensho.step(title, action: nil) { ... }` | Opens a Kensho step, nesting automatically. On exception the step is marked `fail` and the exception re-raises. |
|
|
115
|
+
| `Kensho.attach(path, kind: nil, name: nil, mime_type: nil)` | Copies the file into `kensho-results/attachments/<caseId>/` and registers it on the current case (or current step if one is open). |
|
|
116
|
+
| `Kensho.label(key, value)` | Adds a string label to the case. |
|
|
117
|
+
| `Kensho.link(url, kind: nil, label: nil)` | Adds a hyperlink to the case. |
|
|
118
|
+
| `Kensho.current_case_id` | Returns the stable case id of the running test, or `nil`. |
|
|
119
|
+
|
|
120
|
+
All five are no-ops outside a running example.
|
|
121
|
+
|
|
122
|
+
## Captured stdout / stderr → logs
|
|
123
|
+
|
|
124
|
+
Every `puts` (or `$stderr.write`) the example emits is captured and
|
|
125
|
+
forwarded into `case.logs[]`. The `t` field is currently always `0`
|
|
126
|
+
(per-line offsets are coming).
|
|
127
|
+
|
|
128
|
+
## Status mapping
|
|
129
|
+
|
|
130
|
+
| RSpec outcome | Kensho status |
|
|
131
|
+
| -------------------------- | ------------- |
|
|
132
|
+
| passed | `pass` |
|
|
133
|
+
| failed (Expectation error) | `fail` |
|
|
134
|
+
| failed (other exception) | `broken` |
|
|
135
|
+
| pending / skipped | `skip` |
|
|
136
|
+
|
|
137
|
+
A pending example's `pending_message` becomes the first entry in
|
|
138
|
+
`case.logs[]`.
|
|
139
|
+
|
|
140
|
+
## Environment auto-detected
|
|
141
|
+
|
|
142
|
+
GitHub Actions, CircleCI, GitLab CI, Jenkins, Buildkite, Azure DevOps —
|
|
143
|
+
CI provider, branch, commit, run URL, OS, architecture, Ruby version.
|
|
144
|
+
|
|
145
|
+
Pass `KR_AUTHOR`, `KR_COMMIT_MSG`, `KR_STAGE`, `KR_BASE_URL`,
|
|
146
|
+
`KR_APP_VERSION`, `KR_BUILD_NUMBER`, `KR_RELEASE`, `KR_REGION`,
|
|
147
|
+
`KR_LOCALE`, `KR_TRIGGER`, or `KR_FEATURE` as env vars to populate the
|
|
148
|
+
matching fields on `run.env`.
|
|
149
|
+
|
|
150
|
+
## CLI / env flags
|
|
151
|
+
|
|
152
|
+
There is no formatter-options DSL because RSpec doesn't pass formatter
|
|
153
|
+
arguments through reliably across versions. Configuration goes through
|
|
154
|
+
env vars:
|
|
155
|
+
|
|
156
|
+
```
|
|
157
|
+
KENSHO_OUTPUT output dir (default ./kensho-results)
|
|
158
|
+
KENSHO_PROJECT_NAME project name in run.json
|
|
159
|
+
KENSHO_PROJECT_SLUG project slug
|
|
160
|
+
KENSHO_RUN_ID override the auto-generated run id
|
|
161
|
+
KENSHO_NO_SEVERITY_FROM_META set to "1" to disable severity-from-meta
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
## Design notes
|
|
165
|
+
|
|
166
|
+
- Zero runtime dependencies beyond `rspec-core` itself. The schema lives
|
|
167
|
+
in the JS workspace; we vendor the minimum (id-hashing, status
|
|
168
|
+
mapping, env capture) inline so the gem installs in seconds.
|
|
169
|
+
- The formatter never raises out of a hook — a broken adapter must not
|
|
170
|
+
break a test run.
|
|
171
|
+
- IDs are stable across adapters: the FNV-1a hash matches the JS
|
|
172
|
+
`stableCaseId` byte-for-byte, so `rspec` and `playwright` runs of the
|
|
173
|
+
same suite roll up to the same history on the platform.
|
|
174
|
+
|
|
175
|
+
## License
|
|
176
|
+
|
|
177
|
+
Apache-2.0.
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Vendored slice of the Kensho v1 schema contract.
|
|
4
|
+
#
|
|
5
|
+
# We deliberately do not depend on the JS @kaizenreport/kensho-schema package.
|
|
6
|
+
# Adapters need to stay tiny and standalone, so we re-implement the few things
|
|
7
|
+
# the formatter cares about: the stable case-id hash, the status enum, the
|
|
8
|
+
# attachment kind/MIME tables, and the environment-capture function.
|
|
9
|
+
|
|
10
|
+
require 'rbconfig'
|
|
11
|
+
|
|
12
|
+
module Kensho
|
|
13
|
+
module Schema
|
|
14
|
+
SCHEMA_VERSION = 'kensho/v1'
|
|
15
|
+
|
|
16
|
+
STATUS = %w[pass fail broken skip].freeze
|
|
17
|
+
STEP_STATUS = %w[pass fail skip].freeze
|
|
18
|
+
SEVERITY = %w[blocker critical normal minor trivial].freeze
|
|
19
|
+
ATTACHMENT_KINDS = %w[
|
|
20
|
+
screenshot video trace har text json html dom-snapshot log
|
|
21
|
+
].freeze
|
|
22
|
+
|
|
23
|
+
MIME_BY_EXT = {
|
|
24
|
+
'.png' => 'image/png',
|
|
25
|
+
'.jpg' => 'image/jpeg',
|
|
26
|
+
'.jpeg' => 'image/jpeg',
|
|
27
|
+
'.webp' => 'image/webp',
|
|
28
|
+
'.webm' => 'video/webm',
|
|
29
|
+
'.mp4' => 'video/mp4',
|
|
30
|
+
'.zip' => 'application/zip',
|
|
31
|
+
'.html' => 'text/html',
|
|
32
|
+
'.json' => 'application/json',
|
|
33
|
+
'.txt' => 'text/plain',
|
|
34
|
+
'.log' => 'text/plain',
|
|
35
|
+
'.har' => 'application/json'
|
|
36
|
+
}.freeze
|
|
37
|
+
|
|
38
|
+
KIND_BY_EXT = {
|
|
39
|
+
'.png' => 'screenshot',
|
|
40
|
+
'.jpg' => 'screenshot',
|
|
41
|
+
'.jpeg' => 'screenshot',
|
|
42
|
+
'.webp' => 'screenshot',
|
|
43
|
+
'.webm' => 'video',
|
|
44
|
+
'.mp4' => 'video',
|
|
45
|
+
'.zip' => 'trace',
|
|
46
|
+
'.html' => 'html',
|
|
47
|
+
'.json' => 'json',
|
|
48
|
+
'.txt' => 'text',
|
|
49
|
+
'.log' => 'log',
|
|
50
|
+
'.har' => 'har'
|
|
51
|
+
}.freeze
|
|
52
|
+
|
|
53
|
+
FNV_OFFSET_1 = 0x811c9dc5
|
|
54
|
+
FNV_OFFSET_2 = 0x01000193
|
|
55
|
+
FNV_PRIME_1 = 0x01000193
|
|
56
|
+
FNV_PRIME_2 = 0x85ebca6b
|
|
57
|
+
MASK32 = 0xffffffff
|
|
58
|
+
|
|
59
|
+
# Mirrors stableCaseId from packages/schema/index.js byte-for-byte:
|
|
60
|
+
# double FNV-1a with two different secondary primes so the two 32-bit
|
|
61
|
+
# chunks come from independent rolling states. Must stay byte-compatible
|
|
62
|
+
# with the JS implementation or test history won't line up across
|
|
63
|
+
# adapters.
|
|
64
|
+
def self.stable_case_id(full_name, file_path)
|
|
65
|
+
s = "#{full_name || ''}::#{file_path || ''}"
|
|
66
|
+
h1 = FNV_OFFSET_1
|
|
67
|
+
h2 = FNV_OFFSET_2
|
|
68
|
+
s.each_codepoint do |c|
|
|
69
|
+
h1 = ((h1 ^ c) * FNV_PRIME_1) & MASK32
|
|
70
|
+
h2 = ((h2 ^ c) * FNV_PRIME_2) & MASK32
|
|
71
|
+
end
|
|
72
|
+
format('tc_%08x%08x', h1, h2)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def self.kind_and_mime_for(path)
|
|
76
|
+
ext = File.extname(path.to_s).downcase
|
|
77
|
+
[KIND_BY_EXT[ext] || 'text', MIME_BY_EXT[ext] || 'application/octet-stream']
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def self.normalize_os
|
|
81
|
+
host = RbConfig::CONFIG['host_os'].to_s.downcase
|
|
82
|
+
return 'linux' if host.include?('linux')
|
|
83
|
+
return 'darwin' if host.include?('darwin')
|
|
84
|
+
return 'win32' if host.include?('mswin') || host.include?('mingw') || host.include?('cygwin')
|
|
85
|
+
|
|
86
|
+
host
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Normalize SSH-style git URLs to https. ``git@github.com:foo/bar.git``
|
|
90
|
+
# becomes ``https://github.com/foo/bar``; trailing ``.git`` is dropped.
|
|
91
|
+
def self.normalize_git_url(u)
|
|
92
|
+
return nil if u.nil? || u.empty?
|
|
93
|
+
|
|
94
|
+
m = u.match(%r{^(?:ssh://)?git@([^:/]+)[:/](.+?)(?:\.git)?$})
|
|
95
|
+
return "https://#{m[1]}/#{m[2]}" if m
|
|
96
|
+
|
|
97
|
+
u.sub(/\.git\z/, '')
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# CI / environment metadata for run.env. Matches the helpers in the JS
|
|
101
|
+
# adapters so a Kensho report looks the same regardless of language.
|
|
102
|
+
def self.env_info(framework_version: nil)
|
|
103
|
+
ci_env = ENV['CI']
|
|
104
|
+
ci =
|
|
105
|
+
if ci_env && ENV['GITHUB_ACTIONS']
|
|
106
|
+
'github-actions'
|
|
107
|
+
elsif ci_env && ENV['CIRCLECI']
|
|
108
|
+
'circleci'
|
|
109
|
+
elsif ci_env && ENV['GITLAB_CI']
|
|
110
|
+
'gitlab'
|
|
111
|
+
elsif ci_env && ENV['JENKINS_URL']
|
|
112
|
+
'jenkins'
|
|
113
|
+
elsif ci_env && ENV['BUILDKITE']
|
|
114
|
+
'buildkite'
|
|
115
|
+
elsif ci_env && ENV['TF_BUILD']
|
|
116
|
+
'azure-devops'
|
|
117
|
+
elsif ci_env
|
|
118
|
+
'unknown'
|
|
119
|
+
else
|
|
120
|
+
'local'
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
branch = ENV['GITHUB_REF_NAME'] || ENV['CIRCLE_BRANCH'] ||
|
|
124
|
+
ENV['CI_COMMIT_REF_NAME'] || ENV['BUILDKITE_BRANCH']
|
|
125
|
+
commit = ENV['GITHUB_SHA'] || ENV['CIRCLE_SHA1'] ||
|
|
126
|
+
ENV['CI_COMMIT_SHA'] || ENV['BUILDKITE_COMMIT']
|
|
127
|
+
author = ENV['KR_AUTHOR'] || ENV['GITHUB_ACTOR']
|
|
128
|
+
commit_msg = ENV['KR_COMMIT_MSG']
|
|
129
|
+
|
|
130
|
+
run_url = ENV['KR_RUN_URL'] ||
|
|
131
|
+
if ENV['GITHUB_SERVER_URL'] && ENV['GITHUB_REPOSITORY'] && ENV['GITHUB_RUN_ID']
|
|
132
|
+
"#{ENV['GITHUB_SERVER_URL']}/#{ENV['GITHUB_REPOSITORY']}/actions/runs/#{ENV['GITHUB_RUN_ID']}"
|
|
133
|
+
elsif ENV['CIRCLE_BUILD_URL']
|
|
134
|
+
ENV['CIRCLE_BUILD_URL']
|
|
135
|
+
elsif ENV['CI_JOB_URL']
|
|
136
|
+
ENV['CI_JOB_URL']
|
|
137
|
+
elsif ENV['BUILD_URL']
|
|
138
|
+
ENV['BUILD_URL']
|
|
139
|
+
elsif ENV['BUILDKITE_BUILD_URL']
|
|
140
|
+
ENV['BUILDKITE_BUILD_URL']
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# repoUrl — KR_REPO_URL override → GitHub Actions / GitLab / Bitbucket /
|
|
144
|
+
# Azure / SSH-style URLs from CircleCI / Buildkite / Jenkins (normalized).
|
|
145
|
+
gh_server = ENV['GITHUB_SERVER_URL']
|
|
146
|
+
gh_repo = ENV['GITHUB_REPOSITORY']
|
|
147
|
+
repo_url = ENV['KR_REPO_URL'] ||
|
|
148
|
+
(gh_server && gh_repo ? "#{gh_server}/#{gh_repo}" : nil) ||
|
|
149
|
+
ENV['CI_PROJECT_URL'] ||
|
|
150
|
+
ENV['BITBUCKET_GIT_HTTP_ORIGIN'] ||
|
|
151
|
+
normalize_git_url(ENV['BUILD_REPOSITORY_URI']) ||
|
|
152
|
+
normalize_git_url(ENV['CIRCLE_REPOSITORY_URL'] ||
|
|
153
|
+
ENV['BUILDKITE_REPO'] ||
|
|
154
|
+
ENV['GIT_URL'])
|
|
155
|
+
|
|
156
|
+
info = {
|
|
157
|
+
'ci' => ci,
|
|
158
|
+
'os' => normalize_os,
|
|
159
|
+
'arch' => RbConfig::CONFIG['host_cpu'].to_s
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
ruby_version = "#{RUBY_VERSION}p#{defined?(RUBY_PATCHLEVEL) ? RUBY_PATCHLEVEL : '0'}"
|
|
163
|
+
info['vars'] = { 'rubyVersion' => ruby_version }
|
|
164
|
+
info['vars']['frameworkVersion'] = framework_version if framework_version
|
|
165
|
+
|
|
166
|
+
info['branch'] = branch if branch
|
|
167
|
+
info['commit'] = commit if commit
|
|
168
|
+
info['commitMsg'] = commit_msg if commit_msg
|
|
169
|
+
info['author'] = author if author
|
|
170
|
+
info['runUrl'] = run_url if run_url
|
|
171
|
+
info['repoUrl'] = repo_url if repo_url
|
|
172
|
+
os_version = RbConfig::CONFIG['host_os']
|
|
173
|
+
info['osVersion'] = os_version if os_version
|
|
174
|
+
|
|
175
|
+
[
|
|
176
|
+
['KR_STAGE', 'stage'],
|
|
177
|
+
['KR_BASE_URL', 'baseUrl'],
|
|
178
|
+
['KR_APP_VERSION', 'appVersion'],
|
|
179
|
+
['KR_BUILD_NUMBER', 'buildNumber'],
|
|
180
|
+
['KR_RELEASE', 'release'],
|
|
181
|
+
['KR_REGION', 'region'],
|
|
182
|
+
['KR_LOCALE', 'locale'],
|
|
183
|
+
['KR_TRIGGER', 'trigger'],
|
|
184
|
+
['KR_FEATURE', 'feature']
|
|
185
|
+
].each do |env_var, key|
|
|
186
|
+
v = ENV[env_var]
|
|
187
|
+
info[key] = v if v && !v.empty?
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
info
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
def self.iso_now
|
|
194
|
+
Time.now.utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ')
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
def self.iso_from_seconds(secs)
|
|
198
|
+
Time.at(secs).utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ')
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
def self.slugify(name)
|
|
202
|
+
s = name.to_s.downcase.strip
|
|
203
|
+
s = s.gsub(/[^a-z0-9_-]+/, '-').gsub(/\A-+|-+\z/, '')
|
|
204
|
+
s.empty? ? 'unknown' : s
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
def self.default_run_id
|
|
208
|
+
"run_#{Time.now.utc.strftime('%Y%m%d%H%M%S')}"
|
|
209
|
+
end
|
|
210
|
+
end
|
|
211
|
+
end
|
|
@@ -0,0 +1,559 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
require 'securerandom'
|
|
5
|
+
require 'fileutils'
|
|
6
|
+
require 'stringio'
|
|
7
|
+
require 'set'
|
|
8
|
+
require_relative '../_schema'
|
|
9
|
+
require_relative 'state'
|
|
10
|
+
require_relative 'helpers'
|
|
11
|
+
|
|
12
|
+
# RSpec formatter that emits a Kensho v1 result bundle.
|
|
13
|
+
#
|
|
14
|
+
# Layout written under KENSHO_OUTPUT (default ./kensho-results):
|
|
15
|
+
#
|
|
16
|
+
# kensho-results/
|
|
17
|
+
# run.json manifest (project, env, totals, framework, timing)
|
|
18
|
+
# cases/<stableId>.json one file per example
|
|
19
|
+
# attachments/<caseId>/ files registered via Kensho.attach
|
|
20
|
+
#
|
|
21
|
+
# Register with RSpec:
|
|
22
|
+
#
|
|
23
|
+
# rspec --format Kensho::RSpec::Formatter
|
|
24
|
+
#
|
|
25
|
+
# CLI flags / env vars (the formatter has no rspec-options DSL since
|
|
26
|
+
# RSpec doesn't pass those through formatter args reliably; we use env
|
|
27
|
+
# vars + the `.kensho.yml`-free philosophy of the JS adapters):
|
|
28
|
+
#
|
|
29
|
+
# KENSHO_OUTPUT output directory (default ./kensho-results)
|
|
30
|
+
# KENSHO_PROJECT_NAME project name in run.json
|
|
31
|
+
# KENSHO_PROJECT_SLUG project slug
|
|
32
|
+
# KENSHO_RUN_ID override the auto-generated run id
|
|
33
|
+
# KENSHO_NO_SEVERITY_FROM_META set to "1" to disable severity-from-metadata
|
|
34
|
+
|
|
35
|
+
require 'rspec/core/formatters/base_formatter' unless defined?(::RSpec::Core::Formatters::BaseFormatter)
|
|
36
|
+
|
|
37
|
+
module Kensho
|
|
38
|
+
module RSpec
|
|
39
|
+
class Formatter < ::RSpec::Core::Formatters::BaseFormatter
|
|
40
|
+
::RSpec::Core::Formatters.register self,
|
|
41
|
+
:start,
|
|
42
|
+
:example_group_started,
|
|
43
|
+
:example_started,
|
|
44
|
+
:example_passed,
|
|
45
|
+
:example_failed,
|
|
46
|
+
:example_pending,
|
|
47
|
+
:stop
|
|
48
|
+
|
|
49
|
+
SEVERITY_KEYS = Kensho::Schema::SEVERITY.map(&:to_sym).freeze
|
|
50
|
+
|
|
51
|
+
# Metadata keys that aren't user tags. We prune these from the tag
|
|
52
|
+
# list so RSpec's housekeeping (location, type, file_path, etc.)
|
|
53
|
+
# doesn't pollute case.tags.
|
|
54
|
+
META_BLOCKLIST = %i[
|
|
55
|
+
absolute_file_path block default_path described_class description
|
|
56
|
+
described_class_name described_class_name described_at description_args
|
|
57
|
+
execution_result file_path file_path_proc full_description
|
|
58
|
+
kensho_feature kensho_epic kensho_story
|
|
59
|
+
last_run_status line_number location parent_example_group
|
|
60
|
+
rerun_file_path scoped_id severity shared_group_inclusions
|
|
61
|
+
shared_group_metadata shared_group_inclusion_backtrace
|
|
62
|
+
skip stack_frames type variants
|
|
63
|
+
].to_set
|
|
64
|
+
|
|
65
|
+
def initialize(output = nil)
|
|
66
|
+
# RSpec's BaseFormatter wants an output IO; we never write to it,
|
|
67
|
+
# but we still hand one up so RSpec doesn't blow up.
|
|
68
|
+
super(output || StringIO.new)
|
|
69
|
+
@output_dir = File.expand_path(ENV['KENSHO_OUTPUT'] || (defined?(Kensho::RSpec::DEFAULT_OUTPUT) ? Kensho::RSpec::DEFAULT_OUTPUT : 'kensho-results'))
|
|
70
|
+
@cases_dir = File.join(@output_dir, 'cases')
|
|
71
|
+
@attachments_dir = File.join(@output_dir, 'attachments')
|
|
72
|
+
@project = {
|
|
73
|
+
'name' => ENV['KENSHO_PROJECT_NAME'] || 'Unknown project',
|
|
74
|
+
'slug' => ENV['KENSHO_PROJECT_SLUG'] || Kensho::Schema.slugify(ENV['KENSHO_PROJECT_NAME'] || 'unknown')
|
|
75
|
+
}
|
|
76
|
+
@run_id = ENV['KENSHO_RUN_ID'] || Kensho::Schema.default_run_id
|
|
77
|
+
@severity_from_meta = ENV['KENSHO_NO_SEVERITY_FROM_META'].to_s.empty?
|
|
78
|
+
@started_at = Kensho::Schema.iso_now
|
|
79
|
+
@started_perf = monotonic_now
|
|
80
|
+
@rootpath = Dir.pwd
|
|
81
|
+
|
|
82
|
+
@cases_by_id = {}
|
|
83
|
+
@ids_seen = Hash.new(0)
|
|
84
|
+
@groups = {}
|
|
85
|
+
|
|
86
|
+
FileUtils.mkdir_p(@cases_dir)
|
|
87
|
+
FileUtils.mkdir_p(@attachments_dir)
|
|
88
|
+
|
|
89
|
+
Kensho::RSpec::State.formatter = self
|
|
90
|
+
install_capture_hook!
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Tee stdout/stderr around each example into the per-case scratch so
|
|
94
|
+
# case.logs[] picks up `puts` calls without the user having to do
|
|
95
|
+
# anything. RSpec doesn't expose a "captured output" API for
|
|
96
|
+
# arbitrary formatters, so we install a global before/after hook.
|
|
97
|
+
def install_capture_hook!
|
|
98
|
+
return if @capture_installed
|
|
99
|
+
|
|
100
|
+
@capture_installed = true
|
|
101
|
+
::RSpec.configure do |config|
|
|
102
|
+
config.before(:each) do
|
|
103
|
+
scratch = Kensho::RSpec::State.current
|
|
104
|
+
next unless scratch
|
|
105
|
+
|
|
106
|
+
stdout_io = StringIO.new
|
|
107
|
+
stderr_io = StringIO.new
|
|
108
|
+
scratch.instance_variable_set(:@orig_stdout, $stdout)
|
|
109
|
+
scratch.instance_variable_set(:@orig_stderr, $stderr)
|
|
110
|
+
scratch.instance_variable_set(:@stdout_io, stdout_io)
|
|
111
|
+
scratch.instance_variable_set(:@stderr_io, stderr_io)
|
|
112
|
+
$stdout = stdout_io
|
|
113
|
+
$stderr = stderr_io
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
config.after(:each) do
|
|
117
|
+
scratch = Kensho::RSpec::State.current
|
|
118
|
+
next unless scratch
|
|
119
|
+
|
|
120
|
+
stdout_io = scratch.instance_variable_get(:@stdout_io)
|
|
121
|
+
stderr_io = scratch.instance_variable_get(:@stderr_io)
|
|
122
|
+
orig_out = scratch.instance_variable_get(:@orig_stdout)
|
|
123
|
+
orig_err = scratch.instance_variable_get(:@orig_stderr)
|
|
124
|
+
$stdout = orig_out if orig_out
|
|
125
|
+
$stderr = orig_err if orig_err
|
|
126
|
+
scratch.instance_variable_set(:@stdout_text, stdout_io.string) if stdout_io
|
|
127
|
+
scratch.instance_variable_set(:@stderr_text, stderr_io.string) if stderr_io
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# ----- lifecycle hooks ----- #
|
|
133
|
+
|
|
134
|
+
def start(_notification); end
|
|
135
|
+
|
|
136
|
+
def example_group_started(notification)
|
|
137
|
+
group = notification.group
|
|
138
|
+
@groups[group.metadata[:scoped_id] || group.object_id] = collect_group_meta(group)
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def example_started(notification)
|
|
142
|
+
example = notification.example
|
|
143
|
+
case_obj = build_case(example)
|
|
144
|
+
scratch = CaseScratch.new(
|
|
145
|
+
case_id: case_obj['id'],
|
|
146
|
+
example_id: example.id,
|
|
147
|
+
started_at_ms: (Time.now.to_f * 1000.0)
|
|
148
|
+
)
|
|
149
|
+
scratch.instance_variable_set(:@case_obj, case_obj)
|
|
150
|
+
scratch.instance_variable_set(:@stdout_capture, StringIO.new)
|
|
151
|
+
scratch.instance_variable_set(:@stderr_capture, StringIO.new)
|
|
152
|
+
State.current = scratch
|
|
153
|
+
|
|
154
|
+
@cases_by_id[case_obj['id']] = case_obj
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
def example_passed(notification)
|
|
158
|
+
finalize(notification.example, 'pass', nil)
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
def example_failed(notification)
|
|
162
|
+
ex = notification.example.execution_result.exception
|
|
163
|
+
status =
|
|
164
|
+
if ex && (ex.class.name == 'RSpec::Expectations::ExpectationNotMetError')
|
|
165
|
+
'fail'
|
|
166
|
+
elsif ex
|
|
167
|
+
'broken'
|
|
168
|
+
else
|
|
169
|
+
'fail'
|
|
170
|
+
end
|
|
171
|
+
finalize(notification.example, status, ex)
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def example_pending(notification)
|
|
175
|
+
finalize(notification.example, 'skip', nil)
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
def stop(_notification)
|
|
179
|
+
write_run_json
|
|
180
|
+
ensure
|
|
181
|
+
State.formatter = nil
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# ----- internals ----- #
|
|
185
|
+
|
|
186
|
+
def register_attachment(scratch, src_path, kind:, name:, mime_type:)
|
|
187
|
+
return nil unless File.file?(src_path)
|
|
188
|
+
|
|
189
|
+
attachments_root = File.join(@attachments_dir, scratch.case_id)
|
|
190
|
+
FileUtils.mkdir_p(attachments_root)
|
|
191
|
+
att_id = "att_#{SecureRandom.hex(4)}"
|
|
192
|
+
dest_name = name || File.basename(src_path)
|
|
193
|
+
dest = File.join(attachments_root, "#{att_id}_#{dest_name}")
|
|
194
|
+
begin
|
|
195
|
+
FileUtils.cp(src_path, dest)
|
|
196
|
+
rescue StandardError => e
|
|
197
|
+
warn "[kensho] failed to copy #{src_path}: #{e.message}"
|
|
198
|
+
return nil
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
guessed_kind, guessed_mime = Kensho::Schema.kind_and_mime_for(src_path)
|
|
202
|
+
rel = relpath(dest, @output_dir)
|
|
203
|
+
record = {
|
|
204
|
+
'id' => att_id,
|
|
205
|
+
'kind' => kind || guessed_kind,
|
|
206
|
+
'relativePath' => rel,
|
|
207
|
+
'mimeType' => mime_type || guessed_mime
|
|
208
|
+
}
|
|
209
|
+
begin
|
|
210
|
+
record['sizeBytes'] = File.size(dest)
|
|
211
|
+
rescue StandardError
|
|
212
|
+
# best-effort
|
|
213
|
+
end
|
|
214
|
+
record
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
private
|
|
218
|
+
|
|
219
|
+
def build_case(example)
|
|
220
|
+
full_name = example.full_description
|
|
221
|
+
file_path = relpath(example.metadata[:file_path] || example.metadata[:absolute_file_path], @rootpath)
|
|
222
|
+
line = example.metadata[:line_number]
|
|
223
|
+
|
|
224
|
+
base_id = Kensho::Schema.stable_case_id(full_name, file_path)
|
|
225
|
+
seen = @ids_seen[base_id]
|
|
226
|
+
case_id = seen.zero? ? base_id : "#{base_id}_#{seen + 1}"
|
|
227
|
+
@ids_seen[base_id] = seen + 1
|
|
228
|
+
|
|
229
|
+
suite = collect_suite_chain(example)
|
|
230
|
+
name = example.description.to_s
|
|
231
|
+
|
|
232
|
+
meta = example.metadata
|
|
233
|
+
tags = collect_tags(meta)
|
|
234
|
+
labels = collect_labels(meta)
|
|
235
|
+
links = collect_links(meta)
|
|
236
|
+
params = collect_parameters(meta)
|
|
237
|
+
|
|
238
|
+
severity = collect_severity(meta) if @severity_from_meta
|
|
239
|
+
|
|
240
|
+
behavior = {}
|
|
241
|
+
feature = walk_meta(meta, :kensho_feature) || walk_meta(meta, :feature)
|
|
242
|
+
epic = walk_meta(meta, :kensho_epic) || walk_meta(meta, :epic)
|
|
243
|
+
story = walk_meta(meta, :kensho_story) || walk_meta(meta, :story)
|
|
244
|
+
behavior['feature'] = feature.to_s if feature
|
|
245
|
+
behavior['epic'] = epic.to_s if epic
|
|
246
|
+
behavior['scenario'] = story.to_s if story
|
|
247
|
+
|
|
248
|
+
owner = meta[:owner]
|
|
249
|
+
description = meta[:kensho_description] || meta[:description_text]
|
|
250
|
+
|
|
251
|
+
case_obj = {
|
|
252
|
+
'id' => case_id,
|
|
253
|
+
'name' => name,
|
|
254
|
+
'fullName' => full_name,
|
|
255
|
+
'status' => 'skip',
|
|
256
|
+
'startedAt' => @started_at,
|
|
257
|
+
'duration' => 0,
|
|
258
|
+
'retries' => 0
|
|
259
|
+
}
|
|
260
|
+
case_obj['filePath'] = file_path if file_path
|
|
261
|
+
case_obj['line'] = line.to_i if line
|
|
262
|
+
case_obj['suite'] = suite unless suite.empty?
|
|
263
|
+
case_obj['tags'] = tags unless tags.empty?
|
|
264
|
+
case_obj['severity'] = severity if severity
|
|
265
|
+
case_obj['owner'] = owner.to_s if owner
|
|
266
|
+
case_obj['behavior'] = behavior unless behavior.empty?
|
|
267
|
+
case_obj['labels'] = labels unless labels.empty?
|
|
268
|
+
case_obj['links'] = links unless links.empty?
|
|
269
|
+
case_obj['parameters'] = params unless params.empty?
|
|
270
|
+
case_obj['description'] = description.to_s if description && !description.to_s.empty?
|
|
271
|
+
case_obj['platform'] = Kensho::Schema.normalize_os
|
|
272
|
+
case_obj
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
def finalize(example, status, exception)
|
|
276
|
+
scratch = State.current
|
|
277
|
+
return unless scratch
|
|
278
|
+
|
|
279
|
+
case_obj = scratch.instance_variable_get(:@case_obj)
|
|
280
|
+
result = example.execution_result
|
|
281
|
+
|
|
282
|
+
started_ms = scratch.started_at_ms
|
|
283
|
+
finished_ms = (result.finished_at || Time.now).to_f * 1000.0
|
|
284
|
+
duration_secs = result.run_time
|
|
285
|
+
duration_ms =
|
|
286
|
+
if duration_secs.is_a?(Numeric)
|
|
287
|
+
(duration_secs * 1000.0).round
|
|
288
|
+
else
|
|
289
|
+
(finished_ms - started_ms).round
|
|
290
|
+
end
|
|
291
|
+
duration_ms = 0 if duration_ms.negative?
|
|
292
|
+
|
|
293
|
+
case_obj['status'] = status
|
|
294
|
+
case_obj['startedAt'] = Kensho::Schema.iso_from_seconds(started_ms / 1000.0)
|
|
295
|
+
case_obj['finishedAt'] = Kensho::Schema.iso_from_seconds(finished_ms / 1000.0)
|
|
296
|
+
case_obj['duration'] = duration_ms
|
|
297
|
+
|
|
298
|
+
# Pending/skip messages from RSpec become a log entry so the report
|
|
299
|
+
# surfaces the reason.
|
|
300
|
+
if status == 'skip' && result.pending_message
|
|
301
|
+
case_obj['logs'] ||= []
|
|
302
|
+
case_obj['logs'] << { 't' => 0, 'level' => 'info', 'msg' => result.pending_message.to_s }
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
if exception
|
|
306
|
+
err = { 'message' => first_line(exception.message.to_s) || exception.class.name.to_s }
|
|
307
|
+
stack_text = format_exception(exception)
|
|
308
|
+
err['stack'] = stack_text if stack_text && !stack_text.empty?
|
|
309
|
+
err['type'] = exception.class.name.to_s
|
|
310
|
+
case_obj['errors'] = [err]
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
# Auto-close any steps the user forgot to exit. Mark them broken so
|
|
314
|
+
# the report makes the leak visible.
|
|
315
|
+
until scratch.step_stack.empty?
|
|
316
|
+
leaked = scratch.step_stack.pop
|
|
317
|
+
leaked['status'] ||= 'broken'
|
|
318
|
+
leaked['duration'] ||= 0
|
|
319
|
+
leaked.delete('_started_perf')
|
|
320
|
+
end
|
|
321
|
+
|
|
322
|
+
case_obj['steps'] = scratch.steps unless scratch.steps.empty?
|
|
323
|
+
case_obj['attachments'] = scratch.attachments unless scratch.attachments.empty?
|
|
324
|
+
if !scratch.labels.empty?
|
|
325
|
+
existing = case_obj['labels'] || {}
|
|
326
|
+
case_obj['labels'] = existing.merge(scratch.labels)
|
|
327
|
+
end
|
|
328
|
+
if !scratch.links.empty?
|
|
329
|
+
existing = case_obj['links'] || []
|
|
330
|
+
case_obj['links'] = existing + scratch.links
|
|
331
|
+
end
|
|
332
|
+
|
|
333
|
+
# Stdout/stderr captured by RSpec is wired up by an around-each
|
|
334
|
+
# hook (see Kensho::RSpec::Formatter.install_capture_hook) — see
|
|
335
|
+
# below.
|
|
336
|
+
captured_stdout = scratch.instance_variable_get(:@stdout_text)
|
|
337
|
+
captured_stderr = scratch.instance_variable_get(:@stderr_text)
|
|
338
|
+
capture_logs = []
|
|
339
|
+
if captured_stdout && !captured_stdout.empty?
|
|
340
|
+
captured_stdout.each_line { |ln| capture_logs << { 't' => 0, 'level' => 'info', 'msg' => ln.rstrip } }
|
|
341
|
+
end
|
|
342
|
+
if captured_stderr && !captured_stderr.empty?
|
|
343
|
+
captured_stderr.each_line { |ln| capture_logs << { 't' => 0, 'level' => 'warn', 'msg' => ln.rstrip } }
|
|
344
|
+
end
|
|
345
|
+
unless capture_logs.empty?
|
|
346
|
+
case_obj['logs'] = (case_obj['logs'] || []) + capture_logs
|
|
347
|
+
end
|
|
348
|
+
|
|
349
|
+
write_case(case_obj)
|
|
350
|
+
ensure
|
|
351
|
+
State.current = nil
|
|
352
|
+
end
|
|
353
|
+
|
|
354
|
+
def write_case(case_obj)
|
|
355
|
+
path = File.join(@cases_dir, "#{case_obj['id']}.json")
|
|
356
|
+
File.write(path, JSON.pretty_generate(case_obj))
|
|
357
|
+
rescue StandardError => e
|
|
358
|
+
warn "[kensho] could not write #{File.basename(path)}: #{e.message}"
|
|
359
|
+
end
|
|
360
|
+
|
|
361
|
+
def write_run_json
|
|
362
|
+
finished_at = Kensho::Schema.iso_now
|
|
363
|
+
cases = @cases_by_id.values
|
|
364
|
+
totals = { 'pass' => 0, 'fail' => 0, 'broken' => 0, 'skip' => 0 }
|
|
365
|
+
cases.each { |c| totals[c['status']] += 1 if totals.key?(c['status']) }
|
|
366
|
+
duration_ms = [((monotonic_now - @started_perf) * 1000.0).round, 0].max
|
|
367
|
+
framework_version = (defined?(::RSpec::Core::Version) ? ::RSpec::Core::Version::STRING : 'unknown')
|
|
368
|
+
|
|
369
|
+
run = {
|
|
370
|
+
'schemaVersion' => Kensho::Schema::SCHEMA_VERSION,
|
|
371
|
+
'id' => @run_id,
|
|
372
|
+
'project' => @project,
|
|
373
|
+
'framework' => { 'name' => 'rspec', 'version' => framework_version },
|
|
374
|
+
'env' => Kensho::Schema.env_info(framework_version: framework_version),
|
|
375
|
+
'startedAt' => @started_at,
|
|
376
|
+
'finishedAt' => finished_at,
|
|
377
|
+
'totals' => totals,
|
|
378
|
+
'durationMs' => duration_ms,
|
|
379
|
+
'testCases' => cases
|
|
380
|
+
}
|
|
381
|
+
File.write(File.join(@output_dir, 'run.json'), JSON.pretty_generate(run))
|
|
382
|
+
rescue StandardError => e
|
|
383
|
+
warn "[kensho] failed to write run.json: #{e.message}"
|
|
384
|
+
end
|
|
385
|
+
|
|
386
|
+
# ----- metadata extraction ----- #
|
|
387
|
+
|
|
388
|
+
def collect_severity(meta)
|
|
389
|
+
sev = meta[:severity]
|
|
390
|
+
return sev.to_s if sev.is_a?(String) && Kensho::Schema::SEVERITY.include?(sev.to_s)
|
|
391
|
+
return sev.to_s if sev.is_a?(Symbol) && Kensho::Schema::SEVERITY.include?(sev.to_s)
|
|
392
|
+
|
|
393
|
+
# `it 'foo', :critical do ... end` style — RSpec stores those as
|
|
394
|
+
# `meta[:critical] = true`.
|
|
395
|
+
SEVERITY_KEYS.each do |key|
|
|
396
|
+
return key.to_s if meta[key] == true
|
|
397
|
+
end
|
|
398
|
+
# `severity_blocker: true` style.
|
|
399
|
+
SEVERITY_KEYS.each do |key|
|
|
400
|
+
composite = "severity_#{key}".to_sym
|
|
401
|
+
return key.to_s if meta[composite] == true
|
|
402
|
+
end
|
|
403
|
+
nil
|
|
404
|
+
end
|
|
405
|
+
|
|
406
|
+
def collect_tags(meta)
|
|
407
|
+
tags = []
|
|
408
|
+
meta.each do |k, v|
|
|
409
|
+
next if META_BLOCKLIST.include?(k)
|
|
410
|
+
next if k.to_s.start_with?('rerun_', 'shared_', 'kensho_')
|
|
411
|
+
|
|
412
|
+
if v == true && k.is_a?(Symbol)
|
|
413
|
+
next if SEVERITY_KEYS.include?(k)
|
|
414
|
+
|
|
415
|
+
tags << k.to_s
|
|
416
|
+
end
|
|
417
|
+
end
|
|
418
|
+
# `tags: [...]` explicit array.
|
|
419
|
+
if meta[:tags].is_a?(Array)
|
|
420
|
+
meta[:tags].each { |t| tags << t.to_s unless tags.include?(t.to_s) }
|
|
421
|
+
end
|
|
422
|
+
tags
|
|
423
|
+
end
|
|
424
|
+
|
|
425
|
+
def collect_labels(meta)
|
|
426
|
+
labels = {}
|
|
427
|
+
if meta[:kensho_labels].is_a?(Hash)
|
|
428
|
+
meta[:kensho_labels].each { |k, v| labels[k.to_s] = v.to_s unless v.nil? }
|
|
429
|
+
end
|
|
430
|
+
labels
|
|
431
|
+
end
|
|
432
|
+
|
|
433
|
+
def collect_links(meta)
|
|
434
|
+
links = []
|
|
435
|
+
raw = meta[:kensho_links]
|
|
436
|
+
Array(raw).each do |entry|
|
|
437
|
+
case entry
|
|
438
|
+
when Hash
|
|
439
|
+
url = entry[:url] || entry['url']
|
|
440
|
+
next unless url
|
|
441
|
+
|
|
442
|
+
link = { 'url' => url.to_s }
|
|
443
|
+
kind = entry[:kind] || entry['kind']
|
|
444
|
+
label = entry[:label] || entry['label']
|
|
445
|
+
link['kind'] = kind.to_s if kind
|
|
446
|
+
link['label'] = label.to_s if label
|
|
447
|
+
links << link
|
|
448
|
+
when String
|
|
449
|
+
links << { 'url' => entry }
|
|
450
|
+
end
|
|
451
|
+
end
|
|
452
|
+
links
|
|
453
|
+
end
|
|
454
|
+
|
|
455
|
+
def collect_parameters(meta)
|
|
456
|
+
params = []
|
|
457
|
+
# rspec-parameterized's with_them puts data row in :variants.
|
|
458
|
+
if meta[:variants].is_a?(Hash)
|
|
459
|
+
meta[:variants].each do |k, v|
|
|
460
|
+
params << { 'name' => k.to_s, 'value' => stringify(v), 'kind' => 'data-row' }
|
|
461
|
+
end
|
|
462
|
+
end
|
|
463
|
+
# Conventional `it 'does X', params: { foo: 1 } do` shape.
|
|
464
|
+
if meta[:params].is_a?(Hash)
|
|
465
|
+
meta[:params].each do |k, v|
|
|
466
|
+
next if params.any? { |p| p['name'] == k.to_s }
|
|
467
|
+
|
|
468
|
+
params << { 'name' => k.to_s, 'value' => stringify(v), 'kind' => 'argument' }
|
|
469
|
+
end
|
|
470
|
+
end
|
|
471
|
+
params
|
|
472
|
+
end
|
|
473
|
+
|
|
474
|
+
def collect_group_meta(group)
|
|
475
|
+
m = group.metadata
|
|
476
|
+
{
|
|
477
|
+
feature: m[:kensho_feature] || m[:feature],
|
|
478
|
+
epic: m[:kensho_epic] || m[:epic],
|
|
479
|
+
story: m[:kensho_story] || m[:story]
|
|
480
|
+
}
|
|
481
|
+
end
|
|
482
|
+
|
|
483
|
+
def walk_meta(meta, key)
|
|
484
|
+
return meta[key] if meta[key]
|
|
485
|
+
|
|
486
|
+
parent = meta[:parent_example_group]
|
|
487
|
+
return nil unless parent
|
|
488
|
+
|
|
489
|
+
walk_meta(parent, key)
|
|
490
|
+
end
|
|
491
|
+
|
|
492
|
+
def collect_suite_chain(example)
|
|
493
|
+
chain = []
|
|
494
|
+
group = example.example_group
|
|
495
|
+
if group.respond_to?(:parent_groups)
|
|
496
|
+
# parent_groups[0] is `group` itself, last is the outermost group.
|
|
497
|
+
group.parent_groups.each do |g|
|
|
498
|
+
desc = g.metadata[:description]
|
|
499
|
+
chain.unshift(desc.to_s) if desc && !desc.to_s.empty?
|
|
500
|
+
end
|
|
501
|
+
else
|
|
502
|
+
desc = group.metadata[:description]
|
|
503
|
+
chain << desc.to_s if desc
|
|
504
|
+
end
|
|
505
|
+
chain
|
|
506
|
+
end
|
|
507
|
+
|
|
508
|
+
def relpath(path, root)
|
|
509
|
+
return nil unless path
|
|
510
|
+
|
|
511
|
+
absolute_path = File.absolute_path(path.to_s)
|
|
512
|
+
absolute_root = File.absolute_path(root.to_s)
|
|
513
|
+
if absolute_path.start_with?(absolute_root + File::SEPARATOR)
|
|
514
|
+
absolute_path[(absolute_root.length + 1)..]
|
|
515
|
+
else
|
|
516
|
+
path.to_s.sub(%r{\A\./}, '')
|
|
517
|
+
end
|
|
518
|
+
end
|
|
519
|
+
|
|
520
|
+
def stringify(v)
|
|
521
|
+
case v
|
|
522
|
+
when String then v
|
|
523
|
+
when Symbol then v.to_s
|
|
524
|
+
when Numeric, TrueClass, FalseClass, NilClass then v.inspect
|
|
525
|
+
else
|
|
526
|
+
begin
|
|
527
|
+
v.inspect
|
|
528
|
+
rescue StandardError
|
|
529
|
+
'<unrepr>'
|
|
530
|
+
end
|
|
531
|
+
end
|
|
532
|
+
end
|
|
533
|
+
|
|
534
|
+
def format_exception(exception)
|
|
535
|
+
return nil unless exception
|
|
536
|
+
|
|
537
|
+
lines = [exception.message.to_s]
|
|
538
|
+
if exception.backtrace
|
|
539
|
+
lines.concat(exception.backtrace.first(20))
|
|
540
|
+
end
|
|
541
|
+
lines.join("\n")
|
|
542
|
+
end
|
|
543
|
+
|
|
544
|
+
def first_line(s)
|
|
545
|
+
return nil unless s
|
|
546
|
+
|
|
547
|
+
s.to_s.each_line do |line|
|
|
548
|
+
stripped = line.strip
|
|
549
|
+
return stripped unless stripped.empty?
|
|
550
|
+
end
|
|
551
|
+
nil
|
|
552
|
+
end
|
|
553
|
+
|
|
554
|
+
def monotonic_now
|
|
555
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
556
|
+
end
|
|
557
|
+
end
|
|
558
|
+
end
|
|
559
|
+
end
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'securerandom'
|
|
4
|
+
require 'fileutils'
|
|
5
|
+
|
|
6
|
+
# Public helper API used inside RSpec examples.
|
|
7
|
+
#
|
|
8
|
+
# Kensho.step('open the login page') { ... } # nests automatically
|
|
9
|
+
# Kensho.attach('/tmp/login.png', kind: 'screenshot')
|
|
10
|
+
# Kensho.label('team', 'growth')
|
|
11
|
+
# Kensho.link('https://jira.example.com/browse/PROJ-123', kind: 'jira', label: 'PROJ-123')
|
|
12
|
+
# Kensho.current_case_id # stable id of the running test, or nil
|
|
13
|
+
#
|
|
14
|
+
# All helpers are no-ops outside an example so they're safe to call from
|
|
15
|
+
# shared utility code that may run outside a test context.
|
|
16
|
+
#
|
|
17
|
+
# Top-level Kensho::Feature/Epic/Story sugar lets the spec author tag an
|
|
18
|
+
# example group without typing :metadata twice. They mirror how the
|
|
19
|
+
# pytest plugin exposes @feature/@epic/@story marks.
|
|
20
|
+
|
|
21
|
+
module Kensho
|
|
22
|
+
class << self
|
|
23
|
+
def step(title, action: nil)
|
|
24
|
+
scratch = Kensho::RSpec::State.current
|
|
25
|
+
unless scratch
|
|
26
|
+
return yield({}) if block_given?
|
|
27
|
+
|
|
28
|
+
return {}
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
started_perf = monotonic_now
|
|
32
|
+
step_obj = {
|
|
33
|
+
'id' => "step_#{SecureRandom.hex(5)}",
|
|
34
|
+
'title' => title.to_s,
|
|
35
|
+
'status' => 'pass',
|
|
36
|
+
'startedAt' => Kensho::Schema.iso_now,
|
|
37
|
+
'duration' => 0,
|
|
38
|
+
'_started_perf' => started_perf
|
|
39
|
+
}
|
|
40
|
+
step_obj['action'] = action.to_s if action
|
|
41
|
+
|
|
42
|
+
parent = scratch.step_stack.last
|
|
43
|
+
if parent
|
|
44
|
+
(parent['children'] ||= []) << step_obj
|
|
45
|
+
else
|
|
46
|
+
scratch.steps << step_obj
|
|
47
|
+
end
|
|
48
|
+
scratch.step_stack << step_obj
|
|
49
|
+
|
|
50
|
+
result = nil
|
|
51
|
+
begin
|
|
52
|
+
result = block_given? ? yield(step_obj) : nil
|
|
53
|
+
rescue StandardError, ::RSpec::Expectations::ExpectationNotMetError => e
|
|
54
|
+
step_obj['status'] = 'fail'
|
|
55
|
+
close_step!(step_obj, started_perf)
|
|
56
|
+
scratch.step_stack.pop if scratch.step_stack.last.equal?(step_obj)
|
|
57
|
+
raise e
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
close_step!(step_obj, started_perf)
|
|
61
|
+
scratch.step_stack.pop if scratch.step_stack.last.equal?(step_obj)
|
|
62
|
+
result
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def attach(path, kind: nil, name: nil, mime_type: nil)
|
|
66
|
+
scratch = Kensho::RSpec::State.current
|
|
67
|
+
formatter = Kensho::RSpec::State.formatter
|
|
68
|
+
return nil unless scratch && formatter
|
|
69
|
+
|
|
70
|
+
record = formatter.register_attachment(
|
|
71
|
+
scratch,
|
|
72
|
+
path.to_s,
|
|
73
|
+
kind: kind,
|
|
74
|
+
name: name,
|
|
75
|
+
mime_type: mime_type
|
|
76
|
+
)
|
|
77
|
+
return nil unless record
|
|
78
|
+
|
|
79
|
+
if scratch.step_stack.last
|
|
80
|
+
(scratch.step_stack.last['attachments'] ||= []) << record
|
|
81
|
+
else
|
|
82
|
+
scratch.attachments << record
|
|
83
|
+
end
|
|
84
|
+
record
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def label(key, value)
|
|
88
|
+
scratch = Kensho::RSpec::State.current
|
|
89
|
+
return if scratch.nil? || key.nil? || key.to_s.empty?
|
|
90
|
+
|
|
91
|
+
scratch.labels[key.to_s] = value.to_s
|
|
92
|
+
nil
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def link(url, kind: nil, label: nil)
|
|
96
|
+
scratch = Kensho::RSpec::State.current
|
|
97
|
+
return if scratch.nil? || url.nil? || url.to_s.empty?
|
|
98
|
+
|
|
99
|
+
entry = { 'url' => url.to_s }
|
|
100
|
+
entry['kind'] = kind.to_s if kind
|
|
101
|
+
entry['label'] = label.to_s if label
|
|
102
|
+
scratch.links << entry
|
|
103
|
+
nil
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def current_case_id
|
|
107
|
+
scratch = Kensho::RSpec::State.current
|
|
108
|
+
scratch ? scratch.case_id : nil
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
private
|
|
112
|
+
|
|
113
|
+
def monotonic_now
|
|
114
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def close_step!(step_obj, started_perf)
|
|
118
|
+
step_obj['duration'] = [(((monotonic_now - started_perf) * 1000.0).round), 0].max
|
|
119
|
+
step_obj.delete('_started_perf')
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# Top-level sugar so spec authors can write `Kensho::Feature('Cart')` etc.
|
|
124
|
+
# inside a describe block. These are module methods (uppercase method
|
|
125
|
+
# names are legal in Ruby) that walk the binding to find the example
|
|
126
|
+
# group whose `describe` block is currently executing and set metadata
|
|
127
|
+
# on it.
|
|
128
|
+
def self.Feature(name)
|
|
129
|
+
apply_behavior(:kensho_feature, name)
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def self.Epic(name)
|
|
133
|
+
apply_behavior(:kensho_epic, name)
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def self.Story(name)
|
|
137
|
+
apply_behavior(:kensho_story, name)
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def self.apply_behavior(key, value)
|
|
141
|
+
group = Kensho::RSpec::State.current_group
|
|
142
|
+
return unless group
|
|
143
|
+
|
|
144
|
+
group.metadata[key] = value.to_s
|
|
145
|
+
nil
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
# Lowercase aliases for callers who'd rather not use uppercase methods.
|
|
149
|
+
class << self
|
|
150
|
+
alias_method :feature, :Feature
|
|
151
|
+
alias_method :epic, :Epic
|
|
152
|
+
alias_method :story, :Story
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# Hook into RSpec's example-group definition so we can track the
|
|
157
|
+
# currently-defining group. RSpec calls `subclass` for every nested
|
|
158
|
+
# describe; we wrap it to push/pop a stack on the State module.
|
|
159
|
+
module Kensho
|
|
160
|
+
module RSpec
|
|
161
|
+
# Hook RSpec's example-group definition so Kensho::Feature/Epic/Story
|
|
162
|
+
# can find the group whose describe block is currently executing.
|
|
163
|
+
# We override `module_exec` on the freshly-built subclass to push the
|
|
164
|
+
# class onto a tracking stack while RSpec evaluates the user body.
|
|
165
|
+
module GroupTracker
|
|
166
|
+
def subclass(parent, description, args, registration_collection, &example_group_block)
|
|
167
|
+
wrapped = nil
|
|
168
|
+
if example_group_block
|
|
169
|
+
tracker = Kensho::RSpec::State
|
|
170
|
+
wrapped = lambda do |*lambda_args|
|
|
171
|
+
tracker.push_group(self)
|
|
172
|
+
begin
|
|
173
|
+
instance_exec(*lambda_args, &example_group_block)
|
|
174
|
+
ensure
|
|
175
|
+
tracker.pop_group
|
|
176
|
+
end
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
super(parent, description, args, registration_collection, &(wrapped || example_group_block))
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
if defined?(::RSpec::Core::ExampleGroup)
|
|
186
|
+
::RSpec::Core::ExampleGroup.singleton_class.prepend(Kensho::RSpec::GroupTracker)
|
|
187
|
+
end
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Per-example mutable state used by the public helper API.
|
|
4
|
+
#
|
|
5
|
+
# The formatter sets the current scratch on each example_started and clears
|
|
6
|
+
# it on example_passed/failed/pending. While an example is running, the
|
|
7
|
+
# Kensho.step / Kensho.attach / Kensho.label / Kensho.link helpers mutate
|
|
8
|
+
# the scratch directly so the formatter can pick up the data when it
|
|
9
|
+
# finalizes the case.
|
|
10
|
+
#
|
|
11
|
+
# RSpec runs examples sequentially within a worker by default; we still use
|
|
12
|
+
# Thread.current so threaded fixtures (rare) don't race.
|
|
13
|
+
|
|
14
|
+
module Kensho
|
|
15
|
+
module RSpec
|
|
16
|
+
class CaseScratch
|
|
17
|
+
attr_accessor :case_id, :example_id, :started_at_ms,
|
|
18
|
+
:steps, :step_stack, :attachments, :logs, :labels, :links
|
|
19
|
+
|
|
20
|
+
def initialize(case_id:, example_id:, started_at_ms:)
|
|
21
|
+
@case_id = case_id
|
|
22
|
+
@example_id = example_id
|
|
23
|
+
@started_at_ms = started_at_ms
|
|
24
|
+
@steps = []
|
|
25
|
+
@step_stack = []
|
|
26
|
+
@attachments = []
|
|
27
|
+
@logs = []
|
|
28
|
+
@labels = {}
|
|
29
|
+
@links = []
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
module State
|
|
34
|
+
THREAD_KEY = :__kensho_rspec_scratch__
|
|
35
|
+
GROUP_STACK_KEY = :__kensho_rspec_group_stack__
|
|
36
|
+
|
|
37
|
+
def self.current
|
|
38
|
+
Thread.current[THREAD_KEY]
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def self.current=(scratch)
|
|
42
|
+
Thread.current[THREAD_KEY] = scratch
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def self.push_group(group)
|
|
46
|
+
(Thread.current[GROUP_STACK_KEY] ||= []) << group
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def self.pop_group
|
|
50
|
+
stack = Thread.current[GROUP_STACK_KEY]
|
|
51
|
+
stack&.pop
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def self.current_group
|
|
55
|
+
stack = Thread.current[GROUP_STACK_KEY]
|
|
56
|
+
stack ? stack.last : nil
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
class << self
|
|
60
|
+
attr_accessor :formatter
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
data/lib/kensho/rspec.rb
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Public entrypoint for kensho-rspec.
|
|
4
|
+
#
|
|
5
|
+
# Usage:
|
|
6
|
+
# require 'kensho/rspec'
|
|
7
|
+
# rspec --format Kensho::RSpec::Formatter
|
|
8
|
+
#
|
|
9
|
+
# The helper API (Kensho.step, Kensho.attach, Kensho.label, Kensho.link,
|
|
10
|
+
# Kensho.current_case_id) is wired up here so test code only needs one
|
|
11
|
+
# `require`. All four helpers are no-ops outside a running example, so it
|
|
12
|
+
# is safe to call them from shared utility code.
|
|
13
|
+
|
|
14
|
+
require_relative '_schema'
|
|
15
|
+
require_relative 'rspec/version'
|
|
16
|
+
require_relative 'rspec/state'
|
|
17
|
+
require_relative 'rspec/helpers'
|
|
18
|
+
require_relative 'rspec/formatter'
|
|
19
|
+
|
|
20
|
+
module Kensho
|
|
21
|
+
module RSpec
|
|
22
|
+
DEFAULT_OUTPUT = 'kensho-results'
|
|
23
|
+
end
|
|
24
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: kensho-rspec
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.1
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- KaizenReport
|
|
8
|
+
autorequire:
|
|
9
|
+
bindir: bin
|
|
10
|
+
cert_chain: []
|
|
11
|
+
date: 2026-06-07 00:00:00.000000000 Z
|
|
12
|
+
dependencies:
|
|
13
|
+
- !ruby/object:Gem::Dependency
|
|
14
|
+
name: rspec-core
|
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
|
16
|
+
requirements:
|
|
17
|
+
- - ">="
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: '3.0'
|
|
20
|
+
type: :runtime
|
|
21
|
+
prerelease: false
|
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
23
|
+
requirements:
|
|
24
|
+
- - ">="
|
|
25
|
+
- !ruby/object:Gem::Version
|
|
26
|
+
version: '3.0'
|
|
27
|
+
description: |
|
|
28
|
+
RSpec 3+ formatter that writes a Kensho v1 result bundle (run.json +
|
|
29
|
+
cases/<id>.json + attachments/) into ./kensho-results. Plus a tiny
|
|
30
|
+
helper API (Kensho.step / Kensho.attach / Kensho.label / Kensho.link)
|
|
31
|
+
for surfacing structured metadata from inside examples.
|
|
32
|
+
email:
|
|
33
|
+
executables: []
|
|
34
|
+
extensions: []
|
|
35
|
+
extra_rdoc_files: []
|
|
36
|
+
files:
|
|
37
|
+
- README.md
|
|
38
|
+
- lib/kensho/_schema.rb
|
|
39
|
+
- lib/kensho/rspec.rb
|
|
40
|
+
- lib/kensho/rspec/formatter.rb
|
|
41
|
+
- lib/kensho/rspec/helpers.rb
|
|
42
|
+
- lib/kensho/rspec/state.rb
|
|
43
|
+
- lib/kensho/rspec/version.rb
|
|
44
|
+
homepage: https://github.com/brandon1794/kensho
|
|
45
|
+
licenses:
|
|
46
|
+
- Apache-2.0
|
|
47
|
+
metadata:
|
|
48
|
+
homepage_uri: https://github.com/brandon1794/kensho
|
|
49
|
+
source_code_uri: https://github.com/brandon1794/kensho
|
|
50
|
+
bug_tracker_uri: https://github.com/brandon1794/kensho/issues
|
|
51
|
+
rubygems_mfa_required: 'true'
|
|
52
|
+
post_install_message:
|
|
53
|
+
rdoc_options: []
|
|
54
|
+
require_paths:
|
|
55
|
+
- lib
|
|
56
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
57
|
+
requirements:
|
|
58
|
+
- - ">="
|
|
59
|
+
- !ruby/object:Gem::Version
|
|
60
|
+
version: '2.6'
|
|
61
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
62
|
+
requirements:
|
|
63
|
+
- - ">="
|
|
64
|
+
- !ruby/object:Gem::Version
|
|
65
|
+
version: '0'
|
|
66
|
+
requirements: []
|
|
67
|
+
rubygems_version: 3.5.22
|
|
68
|
+
signing_key:
|
|
69
|
+
specification_version: 4
|
|
70
|
+
summary: Kensho reporter for RSpec — emits Kensho v1 results that the Kensho CLI can
|
|
71
|
+
render into a static HTML report.
|
|
72
|
+
test_files: []
|