@descryy/runtime-test-runner 0.4.12 → 0.4.13

package/reporters/rspec/descry_rspec_formatter.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+
+# §17's RSpec reporter — a custom formatter writing the same NDJSON envelope
+# `reporter.ts`, `jest-reporter.ts`, `vitest-reporter.ts`,
+# `DescryJUnitListener.java` and `descry_pytest_plugin.py` write (RT-108).
+#
+# ## Why a formatter and not the JSON or JUnit output
+#
+# RSpec ships both. `--format json` writes **one document at the end of the
+# run**, so a suite that is killed, hangs, or crashes the process produces
+# nothing at all — the failure mode where evidence matters most. RSpec's
+# JUnit output is a third-party gem and collapses outcomes the way RT-088
+# measured JUnit's own XML doing.
+#
+# The formatter API is RSpec's own event stream. Each example is reported as
+# it finishes, and the outcome is the one RSpec computed.
+#
+# ## Four flavours of not-run, told apart by a snapshot rather than by prose
+#
+# RSpec routes `skip` and `pending` through the same `example_pending`
+# callback, and `example.metadata` by then has been **mutated by execution**.
+# Measured against real rspec-core 3.13.6:
+#
+# | how it was written        | `metadata[:skip]` at `example_started` | at `example_pending` |
+# | ------------------------- | -------------------------------------- | -------------------- |
+# | `it "x", skip: "reason"`  | `"reason"`                             | `"reason"`           |
+# | `xit "x"`                 | `"Temporarily skipped with xit"`       | same                 |
+# | `it "x", skip: true`      | `true`                                 | `true`               |
+# | `skip("reason")` in body  | **`nil`**                              | `true`               |
+#
+# So the metadata at the *end* cannot separate `skip: true` from a runtime
+# `skip` — both read `true` — and the metadata at the *start* can, exactly.
+# That is why this formatter registers `example_started` and snapshots.
+# Parsing `pending_message` for the string RSpec happens to use would be a
+# heuristic over prose, and `xit`'s message would break it first.
+#
+# The mapping that falls out:
+#
+# - declared `skip` (present at start) -> `disabled`, a developer switched it off
+# - runtime `skip` (appears only at the end) -> `precondition-unmet`, it ran
+#   real code and bailed on an assumption. JUnit's `ABORTED` exactly.
+# - `pending`, declared or runtime -> `not-implemented`. The test is written,
+#   it does run, and it is declared not to work yet — node:test's `todo`,
+#   not its `skip`.
+#
+# ## A `pending` example that passes is a failure, and stays one
+#
+# RSpec reports it through `example_failed` with `pending_fixed?` true:
+# "expected to fail but it passed". Forwarded as `test:fail` with the
+# distinction named in the payload, because "the code under test is fixed and
+# nobody removed the marker" is a real finding and silencing it would trade a
+# failure for a silence.
+#
+# ## `example_started` fires for a declared skip, unlike pytest
+#
+# pytest reports a declared skip as a *skipped setup phase*, so no start ever
+# arrives. RSpec starts the example and then reports it pending. Neither is
+# wrong; they are different, and the translator sees both because each
+# reporter reports what its own runner does rather than normalising to the
+# other's shape.
+#
+# ## The envelope version
+#
+# Declared before the first event, as a number: `{"version":"1"}` is read as
+# a broken reporter rather than a matching one. Keep in step with
+# `ENVELOPE_VERSION` in `test-runner-collector.ts` — that file is the
+# authority, not this comment.
+
+require "json"
+
+class DescryRSpecFormatter
+  # Must equal ENVELOPE_VERSION in test-runner-collector.ts. A mismatch is refused there, loudly.
+  ENVELOPE_VERSION = 1
+
+  RSpec::Core::Formatters.register self,
+                                   :start,
+                                   :example_started,
+                                   :example_passed,
+                                   :example_failed,
+                                   :example_pending
+
+  def initialize(output)
+    @output = output
+    # example id -> what `metadata` said BEFORE the example body ran. See the
+    # table in this file's header: the end-state metadata cannot tell a
+    # declared `skip: true` from a runtime `skip`, and this can.
+    @declared = {}
+  end
+
+  def start(_notification)
+    emit({ type: "descry:envelope", data: { version: ENVELOPE_VERSION } })
+  end
+
+  def example_started(notification)
+    example = notification.example
+    @declared[example.id] = {
+      skip: example.metadata[:skip],
+      pending: example.metadata[:pending]
+    }
+    emit({ type: "test:start", data: common(example) })
+  end
+
+  def example_passed(notification)
+    example = notification.example
+    data = common(example)
+    duration = example.execution_result.run_time
+    data[:details] = { duration_ms: duration * 1000 } unless duration.nil?
+    emit({ type: "test:pass", data: data })
+  end
+
+  def example_failed(notification)
+    example = notification.example
+    data = common(example)
+    error = { message: failure_text(notification) }
+    # A `pending` example that passed. Named rather than collapsed into an
+    # ordinary failure: the fix is to delete the marker, not to debug the
+    # code, and only this reporter knows which one it is.
+    error[:failureType] = "pendingFixed" if example.execution_result.pending_fixed?
+    data[:details] = { error: error }
+    emit({ type: "test:fail", data: data })
+  end
+
+  def example_pending(notification)
+    example = notification.example
+    data = common(example)
+    data[:skip] = true
+    data[:skipReason] = skip_reason(example)
+    message = example.execution_result.pending_message
+    data[:message] = message unless message.nil?
+    emit({ type: "test:pass", data: data })
+  end
+
+  private
+
+  def skip_reason(example)
+    declared = @declared[example.id] || {}
+    # Read at the start, before execution mutated it.
+    return "disabled" unless declared[:skip].nil?
+    return "not-implemented" unless declared[:pending].nil?
+    # `metadata[:skip]` appeared during the run: the body executed and called
+    # `skip`. Absent both, it called `pending`, which is the same "known
+    # broken" fact as the declared form.
+    example.metadata[:skip].nil? ? "not-implemented" : "precondition-unmet"
+  end
+
+  def common(example)
+    {
+      name: example.description,
+      # The full `describe`/`context` chain plus the example's own
+      # description — RSpec's own composition, not this formatter's.
+      qualifiedName: example.full_description,
+      file: example.metadata[:file_path],
+      line: example.metadata[:line_number]
+    }
+  end
+
+  def failure_text(notification)
+    exception = notification.exception
+    return notification.message_lines.join("\n") if exception.nil?
+
+    "#{exception.class}: #{exception.message}"
+  end
+
+  # **The leading newline is not cosmetic**, and the reason is measured on
+  # pytest rather than assumed here: a progress formatter writes its dots
+  # with no trailing newline, so a report emitted between two of them lands
+  # on the same line and stops being JSON. A collector then reads every
+  # result as noise while the starts come through, which looks exactly like
+  # a healthy reporter. RSpec runs a single formatter by default and would
+  # not hit it; a developer's own `.rspec` adding `--format progress` would.
+  def emit(object)
+    @output.print("\n#{JSON.generate(object)}\n")
+    @output.flush if @output.respond_to?(:flush)
+  end
+end

package/reporters/xunit/bin/Release/netstandard2.0/Descry.TestLogger.deps.json

@@ -0,0 +1,203 @@
+{
+  "runtimeTarget": {
+    "name": ".NETStandard,Version=v2.0/",
+    "signature": ""
+  },
+  "compilationOptions": {},
+  "targets": {
+    ".NETStandard,Version=v2.0": {},
+    ".NETStandard,Version=v2.0/": {
+      "Descry.TestLogger/1.0.0": {
+        "dependencies": {
+          "Microsoft.TestPlatform.ObjectModel": "17.8.0",
+          "NETStandard.Library": "2.0.3"
+        },
+        "runtime": {
+          "Descry.TestLogger.dll": {}
+        }
+      },
+      "Microsoft.NETCore.Platforms/1.1.0": {},
+      "Microsoft.TestPlatform.ObjectModel/17.8.0": {
+        "dependencies": {
+          "NuGet.Frameworks": "6.5.0",
+          "System.Reflection.Metadata": "1.6.0"
+        },
+        "runtime": {
+          "lib/netstandard2.0/Microsoft.TestPlatform.CoreUtilities.dll": {
+            "assemblyVersion": "15.0.0.0",
+            "fileVersion": "17.800.23.55801"
+          },
+          "lib/netstandard2.0/Microsoft.TestPlatform.PlatformAbstractions.dll": {
+            "assemblyVersion": "15.0.0.0",
+            "fileVersion": "17.800.23.55801"
+          },
+          "lib/netstandard2.0/Microsoft.VisualStudio.TestPlatform.ObjectModel.dll": {
+            "assemblyVersion": "15.0.0.0",
+            "fileVersion": "17.800.23.55801"
+          }
+        },
+        "resources": {
+          "lib/netstandard2.0/cs/Microsoft.TestPlatform.CoreUtilities.resources.dll": {
+            "locale": "cs"
+          },
+          "lib/netstandard2.0/cs/Microsoft.VisualStudio.TestPlatform.ObjectModel.resources.dll": {
+            "locale": "cs"
+          },
+          "lib/netstandard2.0/de/Microsoft.TestPlatform.CoreUtilities.resources.dll": {
+            "locale": "de"
+          },
+          "lib/netstandard2.0/de/Microsoft.VisualStudio.TestPlatform.ObjectModel.resources.dll": {
+            "locale": "de"
+          },
+          "lib/netstandard2.0/es/Microsoft.TestPlatform.CoreUtilities.resources.dll": {
+            "locale": "es"
+          },
+          "lib/netstandard2.0/es/Microsoft.VisualStudio.TestPlatform.ObjectModel.resources.dll": {
+            "locale": "es"
+          },
+          "lib/netstandard2.0/fr/Microsoft.TestPlatform.CoreUtilities.resources.dll": {
+            "locale": "fr"
+          },
+          "lib/netstandard2.0/fr/Microsoft.VisualStudio.TestPlatform.ObjectModel.resources.dll": {
+            "locale": "fr"
+          },
+          "lib/netstandard2.0/it/Microsoft.TestPlatform.CoreUtilities.resources.dll": {
+            "locale": "it"
+          },
+          "lib/netstandard2.0/it/Microsoft.VisualStudio.TestPlatform.ObjectModel.resources.dll": {
+            "locale": "it"
+          },
+          "lib/netstandard2.0/ja/Microsoft.TestPlatform.CoreUtilities.resources.dll": {
+            "locale": "ja"
+          },
+          "lib/netstandard2.0/ja/Microsoft.VisualStudio.TestPlatform.ObjectModel.resources.dll": {
+            "locale": "ja"
+          },
+          "lib/netstandard2.0/ko/Microsoft.TestPlatform.CoreUtilities.resources.dll": {
+            "locale": "ko"
+          },
+          "lib/netstandard2.0/ko/Microsoft.VisualStudio.TestPlatform.ObjectModel.resources.dll": {
+            "locale": "ko"
+          },
+          "lib/netstandard2.0/pl/Microsoft.TestPlatform.CoreUtilities.resources.dll": {
+            "locale": "pl"
+          },
+          "lib/netstandard2.0/pl/Microsoft.VisualStudio.TestPlatform.ObjectModel.resources.dll": {
+            "locale": "pl"
+          },
+          "lib/netstandard2.0/pt-BR/Microsoft.TestPlatform.CoreUtilities.resources.dll": {
+            "locale": "pt-BR"
+          },
+          "lib/netstandard2.0/pt-BR/Microsoft.VisualStudio.TestPlatform.ObjectModel.resources.dll": {
+            "locale": "pt-BR"
+          },
+          "lib/netstandard2.0/ru/Microsoft.TestPlatform.CoreUtilities.resources.dll": {
+            "locale": "ru"
+          },
+          "lib/netstandard2.0/ru/Microsoft.VisualStudio.TestPlatform.ObjectModel.resources.dll": {
+            "locale": "ru"
+          },
+          "lib/netstandard2.0/tr/Microsoft.TestPlatform.CoreUtilities.resources.dll": {
+            "locale": "tr"
+          },
+          "lib/netstandard2.0/tr/Microsoft.VisualStudio.TestPlatform.ObjectModel.resources.dll": {
+            "locale": "tr"
+          },
+          "lib/netstandard2.0/zh-Hans/Microsoft.TestPlatform.CoreUtilities.resources.dll": {
+            "locale": "zh-Hans"
+          },
+          "lib/netstandard2.0/zh-Hans/Microsoft.VisualStudio.TestPlatform.ObjectModel.resources.dll": {
+            "locale": "zh-Hans"
+          },
+          "lib/netstandard2.0/zh-Hant/Microsoft.TestPlatform.CoreUtilities.resources.dll": {
+            "locale": "zh-Hant"
+          },
+          "lib/netstandard2.0/zh-Hant/Microsoft.VisualStudio.TestPlatform.ObjectModel.resources.dll": {
+            "locale": "zh-Hant"
+          }
+        }
+      },
+      "NETStandard.Library/2.0.3": {
+        "dependencies": {
+          "Microsoft.NETCore.Platforms": "1.1.0"
+        }
+      },
+      "NuGet.Frameworks/6.5.0": {
+        "runtime": {
+          "lib/netstandard2.0/NuGet.Frameworks.dll": {
+            "assemblyVersion": "6.5.0.154",
+            "fileVersion": "6.5.0.154"
+          }
+        }
+      },
+      "System.Collections.Immutable/1.5.0": {
+        "runtime": {
+          "lib/netstandard2.0/System.Collections.Immutable.dll": {
+            "assemblyVersion": "1.2.3.0",
+            "fileVersion": "4.6.26515.6"
+          }
+        }
+      },
+      "System.Reflection.Metadata/1.6.0": {
+        "dependencies": {
+          "System.Collections.Immutable": "1.5.0"
+        },
+        "runtime": {
+          "lib/netstandard2.0/System.Reflection.Metadata.dll": {
+            "assemblyVersion": "1.4.3.0",
+            "fileVersion": "4.6.26515.6"
+          }
+        }
+      }
+    }
+  },
+  "libraries": {
+    "Descry.TestLogger/1.0.0": {
+      "type": "project",
+      "serviceable": false,
+      "sha512": ""
+    },
+    "Microsoft.NETCore.Platforms/1.1.0": {
+      "type": "package",
+      "serviceable": true,
+      "sha512": "sha512-kz0PEW2lhqygehI/d6XsPCQzD7ff7gUJaVGPVETX611eadGsA3A877GdSlU0LRVMCTH/+P3o2iDTak+S08V2+A==",
+      "path": "microsoft.netcore.platforms/1.1.0",
+      "hashPath": "microsoft.netcore.platforms.1.1.0.nupkg.sha512"
+    },
+    "Microsoft.TestPlatform.ObjectModel/17.8.0": {
+      "type": "package",
+      "serviceable": true,
+      "sha512": "sha512-AYy6vlpGMfz5kOFq99L93RGbqftW/8eQTqjT9iGXW6s9MRP3UdtY8idJ8rJcjeSja8A18IhIro5YnH3uv1nz4g==",
+      "path": "microsoft.testplatform.objectmodel/17.8.0",
+      "hashPath": "microsoft.testplatform.objectmodel.17.8.0.nupkg.sha512"
+    },
+    "NETStandard.Library/2.0.3": {
+      "type": "package",
+      "serviceable": true,
+      "sha512": "sha512-st47PosZSHrjECdjeIzZQbzivYBJFv6P2nv4cj2ypdI204DO+vZ7l5raGMiX4eXMJ53RfOIg+/s4DHVZ54Nu2A==",
+      "path": "netstandard.library/2.0.3",
+      "hashPath": "netstandard.library.2.0.3.nupkg.sha512"
+    },
+    "NuGet.Frameworks/6.5.0": {
+      "type": "package",
+      "serviceable": true,
+      "sha512": "sha512-QWINE2x3MbTODsWT1Gh71GaGb5icBz4chS8VYvTgsBnsi8esgN6wtHhydd7fvToWECYGq7T4cgBBDiKD/363fg==",
+      "path": "nuget.frameworks/6.5.0",
+      "hashPath": "nuget.frameworks.6.5.0.nupkg.sha512"
+    },
+    "System.Collections.Immutable/1.5.0": {
+      "type": "package",
+      "serviceable": true,
+      "sha512": "sha512-EXKiDFsChZW0RjrZ4FYHu9aW6+P4MCgEDCklsVseRfhoO0F+dXeMSsMRAlVXIo06kGJ/zv+2w1a2uc2+kxxSaQ==",
+      "path": "system.collections.immutable/1.5.0",
+      "hashPath": "system.collections.immutable.1.5.0.nupkg.sha512"
+    },
+    "System.Reflection.Metadata/1.6.0": {
+      "type": "package",
+      "serviceable": true,
+      "sha512": "sha512-COC1aiAJjCoA5GBF+QKL2uLqEBew4JsCkQmoHKbN3TlOZKa2fKLz5CpiRQKDz0RsAOEGsVKqOD5bomsXq/4STQ==",
+      "path": "system.reflection.metadata/1.6.0",
+      "hashPath": "system.reflection.metadata.1.6.0.nupkg.sha512"
+    }
+  }
+}