gitlab-labkit 1.1.2 → 1.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a3f875b5fa4af36bdb6646a69d5acab871bd2fe84ee6876a57958e9264441378
-  data.tar.gz: f86e176c5bed6fcbeae0ac82570b7b98349a56dac3b032cb0072573206ea273e
+  metadata.gz: 1bbb89ac1275f06d828f9d9fa4a24e20e33f4a01d201956d4dd8d30e87dd8f5a
+  data.tar.gz: 1d6b24a17f8b2096e34303fba306ea1271146b3245cb5edcb3fabbccd26f1e45
 SHA512:
-  metadata.gz: d47772da83bcf81c149e957c4e692042ce1388dc774abbfcf9e9d3a765fa9fda88da1b8742314df3b1010e800935327cf4338c0f7ef05f6d2141d8aad3cb317d
-  data.tar.gz: fc070e4ef55f7fac9bedf2af11a88aa5c27aff9c8cae3d634b78a3381e2694a67871a7c16c7865950ad3f7d7371c10cc14e1bbeb38c9bca8f36d00c68717a70c
+  metadata.gz: 4950f9c1615b432b5d53ab54a51cca84a10cbf548b434665fdda93a2204f912f172540652df7b634907c61fa88f03f50a1ad9de96e04d2795a59393fe3fdcf93
+  data.tar.gz: a67c2262dd119373431f271779baf8311c5eb47a794a2970e1b4b9dcd28c59a4253ff844654f20a99026f95aabe6538f8fbdc864da29f007cd6c94a99eb427ad

data/doc/architecture/decisions/001_field_standardization_dynamic_runtime_linting.md

@@ -0,0 +1,124 @@
+---
+owning-stage: '~devops::developer experience'
+description: 'Logging Field Standardization: Dynamic Runtime Linting'
+---
+
+# Logging Field Standardization ADR: Dynamic Runtime Linting
+
+## Context
+
+GitLab is implementing a [logging field standardization initiative](https://handbook.gitlab.com/handbook/engineering/architecture/design-documents/observability_field_standardisation/) to ensure logs are queryable and actionable across all systems. We need a mechanism to identify and track deprecated or non-standard logging fields across multiple codebases.
+
+### Requirements
+
+The solution must:
+
+1. Detect deprecated fields at the point of emission (runtime).
+1. Provide immediate feedback to developers (shift-left).
+1. Prevent new violations without blocking existing work.
+1. Support gradual migration from deprecated to standard fields.
+1. Never affect production performance.
+1. Provide clear guidance on standard field replacements.
+
+### The problem with static analysis
+
+We initially considered RuboCop (static linting), but this approach is inadequate due to Ruby's [dynamic and complex field construction](https://gitlab.com/gitlab-org/gitlab/-/blob/cfd4fb97968d1f7d30f39f89740e414e9437063a/lib/bulk_imports/logger.rb#L46).
+
+Static analysis can't reliably detect:
+
+- Fields merged from hash arguments.
+- Dynamically constructed field names.
+- Fields passed through multiple abstraction layers.
+- Conditional field inclusion based on runtime state.
+
+## Decision
+
+Implement dynamic runtime linting to validate logging fields as they're emitted during development and testing, rather than using static analysis.
+
+The validator will:
+
+- Intercept logging calls at runtime to detect deprecated fields.
+- Compare detected violations against a frozen baseline of known violations.
+- Raise an error on new violations while ignoring tracked existing tracked violations.
+- Provide immediate feedback to developers during local development.
+- Never affect production performance.
+
+## Consequences
+
+### Benefits
+
+- **Shift-left feedback**: Developers discover violations during local development, not in code review.
+- **Comprehensive detection**: Captures violations from all code execution paths (tests, Rake tasks, console, development environments).
+- **Accurate detection**: Runtime interception catches dynamically constructed fields that static analysis misses.
+- **Non-blocking**: Existing violations don't prevent development. They're tracked explicitly in baseline.
+- **Regression prevention**: CI fails on new violations, preventing deprecated fields from reappearing.
+- **Clear guidance**: Exact replacement field suggested for each violation.
+- **Zero production impact**: Validation only runs in development and test environments.
+
+### Trade-offs
+
+- **Execution path dependency**: Only detects violations in code paths that execute during the process.
+- **Runtime overhead**: Adds interception to all non-production processes.
+- **At-exit reporting**: Violations not visible until process completes.
+- **Baseline maintenance**: YAML files require updates when violations are fixed or added.
+- **Learning curve**: Developers must understand baseline management.
+- **File-level scoping**: Multiple violations in the same file are tracked together, making partial fixes during test runs challenging.
+
+### Risks and mitigations
+
+| Risk | Mitigation |
+|------|------------|
+| Limited code path coverage misses violations | Use Kibana to identify fields in production logs, run comprehensive test suites, use development server testing |
+| Baseline drift across branches | Clear documentation, automated baseline regeneration support |
+| At-exit reporting missed if process crashes | Violations still prevented in CI where processes complete successfully |
+
+## Alternatives
+
+### Alternative 1: Static analysis with RuboCop
+
+Use custom RuboCop cops to detect deprecated field usage.
+
+Rejected because:
+
+- Can't handle fields merged from hash variables.
+- Would produce many false negatives.
+- Complex pattern matching still misses dynamic cases.
+- Poor developer experience with unclear violations.
+
+### Alternative 2: Manual tracking
+
+Track violations in spreadsheets or GitLab issues.
+
+Rejected because:
+
+- No automated enforcement or detection.
+- Manual process becomes stale quickly.
+- No shift-left feedback to developers.
+- Can't prevent regressions.
+
+### Alternative 3: Grep-based CI checks
+
+Search source code for deprecated field strings in CI.
+
+Rejected because:
+
+- High false positive rate (matches in comments, strings, tests).
+- Can't distinguish field usage from definitions.
+- No stable tracking across refactoring.
+- Poor user experience with unclear error messages.
+
+### Alternative 4: Production log analysis only
+
+Rely solely on Kibana analysis to find deprecated fields.
+
+Rejected because:
+
+- No prevention, only reactive detection.
+- Violations already in production when discovered.
+- No developer feedback during development.
+- Difficult to trace back to specific code locations.
+
+## References
+
+- [Parent Epic](https://gitlab.com/groups/gitlab-org/quality/-/work_items/235)
+- [Observability Field Standardisation](https://handbook.gitlab.com/handbook/engineering/architecture/design-documents/observability_field_standardisation/)

data/gitlab-labkit.gemspec

@@ -29,7 +29,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "openssl", "~> 3.3.2"
   spec.add_runtime_dependency "opentracing", "~> 0.4"
   spec.add_runtime_dependency "pg_query", ">= 6.1.0", "< 7.0"
-  spec.add_runtime_dependency "prometheus-client-mmap", "~> 1.2.9"
+  spec.add_runtime_dependency "prometheus-client-mmap", ">= 1.2", "< 2.0"
   spec.add_runtime_dependency "redis", "> 3.0.0", "< 6.0.0"
 
   # Please maintain alphabetical order for dev dependencies

data/lib/labkit/logging/json_logger.rb

@@ -38,6 +38,12 @@ module Labkit
       end
 
       def format_message(severity, timestamp, progname, message)
+        data = format_data(severity, timestamp, progname, message)
+
+        dump_json(data) << "\n"
+      end
+
+      def format_data(severity, timestamp, progname, message)
         data = default_attributes
         data[:severity] = severity
         data[:time] = timestamp.utc.iso8601(3)
@@ -53,11 +59,10 @@ module Labkit
           data[:message] = message
         when Hash
           reject_reserved_log_keys!(message)
-          format_time!(message)
           data.merge!(message)
         end
 
-        dump_json(data) << "\n"
+        data
       end
 
       private
@@ -79,28 +84,6 @@ module Labkit
                   "\n\nUse key names that are descriptive e.g. by using a prefix."
         end
       end
-
-      def format_time!(hash)
-        hash.each do |key, value|
-          hash[key] = convert_time_value(value)
-        end
-      end
-
-      def convert_time_value(value)
-        case value
-        when Time
-          value.utc.iso8601(3)
-        when DateTime
-          value.to_time.utc.iso8601(3)
-        when Hash
-          format_time!(value)
-          value
-        when Array
-          value.map { |v| convert_time_value(v) }
-        else
-          value
-        end
-      end
     end
   end
 end

data/lib/labkit/user_experience_sli/experience.rb

@@ -236,9 +236,9 @@ module Labkit
           user_experience_id: id,
           feature_category: @definition.feature_category,
           urgency: @definition.urgency,
-          start_time: @start_time,
-          checkpoint_time: @checkpoint_time,
-          end_time: @end_time,
+          start_time: @start_time&.iso8601(3),
+          checkpoint_time: @checkpoint_time&.iso8601(3),
+          end_time: @end_time&.iso8601(3),
           elapsed_time_s: elapsed_time,
           urgency_threshold_s: urgency_threshold
         )

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-labkit
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.1.3
 platform: ruby
 authors:
 - Andrew Newdigate
@@ -169,16 +169,22 @@ dependencies:
   name: prometheus-client-mmap
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.2.9
+        version: '1.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.2.9
+        version: '1.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
@@ -475,6 +481,7 @@ files:
 - Rakefile
 - config/user_experience_slis/schema.json
 - config/user_experience_slis/testing_sample.yml
+- doc/architecture/decisions/001_field_standardization_dynamic_runtime_linting.md
 - gitlab-labkit.gemspec
 - lib/gitlab-labkit.rb
 - lib/labkit/context.rb