bitfab 0.33.5 → 0.34.0

data/lib/bitfab/otel.rb

@@ -0,0 +1,717 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require "time"
+require "opentelemetry/sdk"
+
+require_relative "serialize"
+require_relative "version"
+require_relative "warn_once"
+
+module Bitfab
+  # OpenTelemetry delivery for Bitfab spans and trace completions.
+  #
+  # Bitfab payloads ride as OTLP span attributes (bitfab.operation +
+  # bitfab.payload), so a batch travels to Bitfab as OTLP/JSON without the
+  # payload itself changing shape.
+  module Otel
+    OPERATION_ATTRIBUTE = "bitfab.operation"
+    PAYLOAD_ATTRIBUTE = "bitfab.payload"
+    OTLP_TRACES_ENDPOINT = "/api/sdk/otel/v1/traces"
+    MAX_REQUEST_BYTES_ENV = "BITFAB_OTEL_MAX_REQUEST_BYTES"
+    EXPORT_CONCURRENCY_ENV = "BITFAB_OTEL_EXPORT_CONCURRENCY"
+    MAX_EXPORT_REQUEST_BYTES = 3_000_000
+    MAX_QUEUE_SIZE = 8_192
+    DIRECT_MAX_EXPORT_BATCH_SIZE = 512
+    DIRECT_MAX_REQUEST_BATCH_SIZE = 8
+    DEFAULT_EXPORT_CONCURRENCY = 32
+    MAX_EXPORT_CONCURRENCY = 64
+    SCHEDULE_DELAY_MILLIS = 5_000
+    EXPORT_TIMEOUT_MILLIS = 30_000
+    EXPORT_RETRIES = 3
+    RETRY_DELAY_SECONDS = 0.1
+
+    # The comma that joins adjacent spans in the request's span list.
+    SPAN_SEPARATOR_BYTES = 1
+
+    SUCCESS = OpenTelemetry::SDK::Trace::Export::SUCCESS
+    FAILURE = OpenTelemetry::SDK::Trace::Export::FAILURE
+
+    # A span encoded exactly as it goes on the wire, carrying its byte count.
+    # Encoding once and remembering the size is what keeps request packing
+    # linear: sizing a candidate batch by re-encoding the whole request
+    # re-escapes every carrier's bitfab.payload string on every span considered.
+    EncodedSpan = Struct.new(:encoded, :size)
+
+    # The invariant head and tail of an OTLP request for one export window. Key
+    # order matches what JSON.generate emits for the equivalent Hash, so a body
+    # assembled by concatenation is byte-identical to encoding that Hash.
+    RequestEnvelope = Struct.new(:head, :tail, :size)
+
+    RequestBatch = Struct.new(:spans, :size)
+
+    SPAN_KINDS = {
+      internal: 1,
+      server: 2,
+      client: 3,
+      producer: 4,
+      consumer: 5
+    }.freeze
+
+    INVALID_SPAN_ID = ("\0" * 8).b
+
+    PayloadTooLargeError = Class.new(StandardError)
+    PartialSuccessError = Class.new(StandardError)
+
+    class << self
+      def max_request_bytes_from_env
+        raw = ENV[MAX_REQUEST_BYTES_ENV]
+        return MAX_EXPORT_REQUEST_BYTES if raw.nil?
+
+        value = Integer(raw, exception: false) || 0
+        return value if value.positive? && value <= MAX_EXPORT_REQUEST_BYTES
+
+        Bitfab.warn_once(
+          "otel-max-request-bytes-invalid",
+          "#{MAX_REQUEST_BYTES_ENV} must be a positive integer no greater than " \
+          "#{MAX_EXPORT_REQUEST_BYTES}; using #{MAX_EXPORT_REQUEST_BYTES}"
+        )
+        MAX_EXPORT_REQUEST_BYTES
+      end
+
+      def export_concurrency_from_env
+        raw = ENV[EXPORT_CONCURRENCY_ENV]
+        return DEFAULT_EXPORT_CONCURRENCY if raw.nil?
+
+        value = Integer(raw, exception: false) || 0
+        return value if value.positive? && value <= MAX_EXPORT_CONCURRENCY
+
+        Bitfab.warn_once(
+          "otel-export-concurrency-invalid",
+          "#{EXPORT_CONCURRENCY_ENV} must be a positive integer no greater than " \
+          "#{MAX_EXPORT_CONCURRENCY}; using #{DEFAULT_EXPORT_CONCURRENCY}"
+        )
+        DEFAULT_EXPORT_CONCURRENCY
+      end
+
+      def create_transport(direct_sender:)
+        BatchTransport.new(
+          direct_sender:,
+          export_concurrency: export_concurrency_from_env,
+          max_request_bytes: max_request_bytes_from_env
+        )
+      end
+
+      def register_transport(transport)
+        ensure_process_state
+        live_transports_mutex.synchronize { live_transports << transport }
+      end
+
+      def unregister_transport(transport)
+        ensure_process_state
+        live_transports_mutex.synchronize { live_transports.delete(transport) }
+      end
+
+      def flush_transports(timeout = 30.0)
+        ensure_process_state
+        deadline = monotonic_now + [timeout, 0].max
+        current_process_transports.reduce(true) do |succeeded, transport|
+          transport.flush([deadline - monotonic_now, 0].max) && succeeded
+        end
+      end
+
+      def shutdown_transports(timeout = 30.0)
+        ensure_process_state
+        deadline = monotonic_now + [timeout, 0].max
+        current_process_transports.reduce(true) do |succeeded, transport|
+          transport.shutdown([deadline - monotonic_now, 0].max) && succeeded
+        end
+      end
+
+      # Count the spans submitted for each replay trace and forget them, so the
+      # replay barrier knows how many spans the server must have persisted
+      # before the run can be finalized.
+      def take_replay_span_counts(trace_ids)
+        ensure_process_state
+        submission_mutex.synchronize do
+          counts = trace_ids.each_with_object({}) do |trace_id, acc|
+            next unless replay_trace_submissions.include?(trace_id)
+
+            acc[trace_id] = (trace_submission_span_ids.delete(trace_id) || Set.new).size
+          end
+          replay_trace_submissions.subtract(trace_ids)
+          counts
+        end
+      end
+
+      def record_submission(operation, payload)
+        ensure_process_state
+        source_trace_id = payload["sourceTraceId"]
+        unless source_trace_id.is_a?(String)
+          raw_trace = payload["externalTrace"] || payload["rawTrace"]
+          source_trace_id = raw_trace["id"] if raw_trace.is_a?(Hash)
+        end
+        return unless source_trace_id.is_a?(String)
+
+        submission_mutex.synchronize do
+          if operation == "external_span"
+            record_span_submission(source_trace_id, payload)
+          elsif payload["completed"] == true
+            record_trace_completion(source_trace_id, payload)
+          end
+        end
+      end
+
+      def monotonic_now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
+      # Ruby has no after-fork hook a library can register (Python's
+      # os.register_at_fork has no equivalent), so every entry point that
+      # touches module state checks the pid first. Inherited mutexes need no
+      # special handling: CRuby abandons a mutex held by a thread the child did
+      # not inherit, so the child sees it unlocked.
+      def ensure_process_state
+        @state_pid ||= Process.pid
+        return if @state_pid == Process.pid
+
+        reset_state_after_fork
+      end
+
+      # Replay submission counts belong to the parent's run, never the child's.
+      def reset_state_after_fork
+        @state_pid = Process.pid
+        @trace_submission_span_ids = {}
+        @replay_trace_submissions = Set.new
+      end
+
+      private
+
+      def record_span_submission(source_trace_id, payload)
+        raw_span = payload["rawSpan"]
+        span_id = raw_span["id"] if raw_span.is_a?(Hash)
+        span_id = "submission-#{monotonic_now}" unless span_id.is_a?(String)
+        (trace_submission_span_ids[source_trace_id] ||= Set.new) << span_id
+      end
+
+      def record_trace_completion(source_trace_id, payload)
+        if payload["testRunId"].is_a?(String)
+          replay_trace_submissions << source_trace_id
+          trace_submission_span_ids[source_trace_id] ||= Set.new
+        else
+          trace_submission_span_ids.delete(source_trace_id)
+        end
+      end
+
+      def current_process_transports
+        live_transports_mutex.synchronize { live_transports.select { |t| t.owner_pid == Process.pid } }
+      end
+
+      def live_transports
+        @live_transports ||= Set.new
+      end
+
+      def live_transports_mutex
+        @live_transports_mutex ||= Mutex.new
+      end
+
+      def submission_mutex
+        @submission_mutex ||= Mutex.new
+      end
+
+      def trace_submission_span_ids
+        @trace_submission_span_ids ||= {}
+      end
+
+      def replay_trace_submissions
+        @replay_trace_submissions ||= Set.new
+      end
+    end
+
+    # Encodes carrier spans as OTLP/JSON and posts them straight to Bitfab.
+    class DirectExporter
+      def initialize(direct_sender:, max_request_bytes:, max_request_batch_size:, export_concurrency:)
+        @direct_sender = direct_sender
+        @max_request_bytes = max_request_bytes
+        @max_request_batch_size = max_request_batch_size
+        @export_concurrency = export_concurrency
+      end
+
+      def export(spans, timeout: nil)
+        spans = spans.to_a
+        return SUCCESS if spans.empty?
+
+        begin
+          encoded = spans.map { |span| Encoder.encode_span(span) }
+          envelope = Encoder.request_envelope(spans.first)
+        rescue => e
+          Bitfab.warn_once(
+            "otel-encode-failed",
+            "failed to encode an OpenTelemetry span batch (#{e.message})"
+          )
+          return FAILURE
+        end
+
+        batches = build_request_batches(envelope, encoded)
+        results = send_batches(envelope, batches)
+        results.all? ? SUCCESS : FAILURE
+      end
+
+      def force_flush(timeout: nil)
+        SUCCESS
+      end
+
+      def shutdown(timeout: nil)
+        SUCCESS
+      end
+
+      private
+
+      def send_batches(envelope, batches)
+        results = Array.new(batches.size, false)
+        next_index = 0
+        index_mutex = Mutex.new
+
+        worker_count = [@export_concurrency, batches.size].min
+        workers = Array.new(worker_count) do
+          Thread.new do
+            loop do
+              index = index_mutex.synchronize do
+                current = next_index
+                next_index += 1 if current < batches.size
+                current
+              end
+              break if index >= batches.size
+
+              results[index] = send_batch(envelope, batches[index])
+            end
+          end
+        end
+        workers.each(&:join)
+        results
+      end
+
+      def build_request_batches(envelope, encoded)
+        batches = []
+        current = []
+        size = envelope.size
+
+        encoded.each do |span|
+          addition = span.size + (current.empty? ? 0 : SPAN_SEPARATOR_BYTES)
+          if !current.empty? && (current.size >= @max_request_batch_size || size + addition > @max_request_bytes)
+            batches << RequestBatch.new(current, size)
+            current = []
+            size = envelope.size
+            addition = span.size
+          end
+          current << span
+          size += addition
+        end
+
+        batches << RequestBatch.new(current, size) unless current.empty?
+        batches
+      end
+
+      def send_batch(envelope, batch)
+        if batch.size > @max_request_bytes
+          warn_oversized(batch.spans.size)
+          return false
+        end
+
+        send_with_retries(Encoder.encode_request(envelope, batch.spans))
+        true
+      rescue PayloadTooLargeError
+        warn_oversized(batch.spans.size)
+        false
+      rescue PartialSuccessError
+        false
+      rescue => e
+        Bitfab.warn_once(
+          "otel-export-failed",
+          "failed to export an OpenTelemetry span batch (further occurrences " \
+          "suppressed): #{e.message}"
+        )
+        false
+      end
+
+      def warn_oversized(span_count)
+        subject = (span_count == 1) ? "a single OpenTelemetry span" : "an OpenTelemetry span batch"
+        Bitfab.warn_once(
+          "otel-payload-too-large",
+          "#{subject} exceeded the ingestion request limit and could not be exported"
+        )
+      end
+
+      def send_with_retries(body)
+        attempt = 0
+        begin
+          attempt += 1
+          response = @direct_sender.call(OTLP_TRACES_ENDPOINT, body, EXPORT_TIMEOUT_MILLIS / 1000.0)
+          check_partial_success(response)
+        rescue PartialSuccessError
+          raise
+        rescue => e
+          raise PayloadTooLargeError if response_status(e) == 413
+          raise if attempt >= EXPORT_RETRIES || !retryable?(e)
+
+          sleep(RETRY_DELAY_SECONDS)
+          retry
+        end
+      end
+
+      def check_partial_success(response)
+        partial_success = response.is_a?(Hash) ? response["partialSuccess"] : nil
+        return unless partial_success.is_a?(Hash)
+
+        rejected = partial_success["rejectedSpans"]
+        return if rejected.nil? || rejected.to_s == "0"
+
+        Bitfab.warn_once(
+          "otel-partial-success",
+          "OTLP ingestion rejected #{rejected} span(s): " \
+          "#{partial_success["errorMessage"] || "no reason provided"}"
+        )
+        raise PartialSuccessError
+      end
+
+      def response_status(error)
+        return nil unless error.respond_to?(:response)
+
+        response = error.response
+        response.respond_to?(:code) ? response.code.to_i : nil
+      end
+
+      def retryable?(error)
+        status = response_status(error)
+        return true if status.nil?
+
+        status >= 500 || [408, 425, 429].include?(status)
+      end
+    end
+
+    # Records failed exports so a flush can report delivery, not just drain.
+    class DeliveryTrackingExporter
+      def initialize(exporter)
+        @exporter = exporter
+        @mutex = Mutex.new
+        @failed_exports = 0
+      end
+
+      def export(spans, timeout: nil)
+        result = begin
+          @exporter.export(spans, timeout:)
+        rescue
+          @mutex.synchronize { @failed_exports += 1 }
+          raise
+        end
+        @mutex.synchronize { @failed_exports += 1 } unless result == SUCCESS
+        result
+      end
+
+      def take_failed_exports
+        @mutex.synchronize do
+          failed = @failed_exports
+          @failed_exports = 0
+          failed
+        end
+      end
+
+      def force_flush(timeout: nil)
+        @exporter.force_flush(timeout:)
+      end
+
+      def shutdown(timeout: nil)
+        @exporter.shutdown(timeout:)
+      end
+    end
+
+    # Owns one private TracerProvider + BatchSpanProcessor per client. The
+    # provider is never installed globally, so the SDK cannot disturb an
+    # application's own OpenTelemetry setup.
+    class BatchTransport
+      attr_reader :owner_pid
+
+      def initialize(direct_sender:, max_export_batch_size: nil,
+        max_request_batch_size: DIRECT_MAX_REQUEST_BATCH_SIZE, max_queue_size: MAX_QUEUE_SIZE,
+        export_concurrency: DEFAULT_EXPORT_CONCURRENCY, max_request_bytes: nil)
+        raise ArgumentError, "max_request_batch_size must be a positive integer" unless max_request_batch_size.positive?
+
+        @direct_sender = direct_sender
+        @max_export_batch_size = max_export_batch_size || DIRECT_MAX_EXPORT_BATCH_SIZE
+        @max_request_batch_size = max_request_batch_size
+        @max_queue_size = max_queue_size
+        @export_concurrency = export_concurrency
+        @max_request_bytes = max_request_bytes || MAX_EXPORT_REQUEST_BYTES
+        @owner_pid = Process.pid
+        @state_mutex = Mutex.new
+        @flush_mutex = Mutex.new
+        @closed = false
+        create_pipeline
+        Otel.register_transport(self)
+      end
+
+      def submit(operation, payload)
+        Otel.record_submission(operation, payload)
+        # Encoding is the expensive part of a submit and needs no mutual
+        # exclusion, so it stays outside the lock.
+        name = Encoder.span_name(operation, payload)
+        encoded_payload = Serialize.safe_generate(payload)
+        started_at = Encoder.timestamp(payload, "started_at")
+        ended_at = Encoder.timestamp(payload, "ended_at")
+        errored = Encoder.error?(payload)
+
+        # Pipeline construction and the closed check are serialized: concurrent
+        # first submits in a forked child would otherwise each build a pipeline,
+        # orphaning one batch worker along with whatever it had queued.
+        @state_mutex.synchronize do
+          return warn_closed if @closed
+
+          ensure_process
+          tracer = @tracer
+          return warn_closed if tracer.nil?
+
+          span = tracer.start_root_span(
+            name,
+            attributes: {
+              OPERATION_ATTRIBUTE => operation,
+              PAYLOAD_ATTRIBUTE => encoded_payload
+            },
+            start_timestamp: started_at
+          )
+          span.status = OpenTelemetry::Trace::Status.error if errored
+          span.finish(end_timestamp: ended_at)
+        end
+        nil
+      rescue => e
+        Bitfab.warn_once(
+          "otel-submit-failed",
+          "failed to queue an OpenTelemetry span (further occurrences " \
+          "suppressed): #{e.message}"
+        )
+        nil
+      end
+
+      def flush(timeout = 30.0)
+        processor, delivery_tracker = @state_mutex.synchronize do
+          ensure_process
+          [@processor, @delivery_tracker]
+        end
+        return true if processor.nil?
+
+        flushed = false
+        flush_thread = Thread.new do
+          @flush_mutex.synchronize do
+            drained = begin
+              processor.force_flush(timeout:)
+            rescue => e
+              Bitfab.warn_once(
+                "otel-flush-failed",
+                "failed to flush OpenTelemetry spans (further occurrences " \
+                "suppressed): #{e.message}"
+              )
+              FAILURE
+            end
+            failed_exports = delivery_tracker.nil? ? 0 : delivery_tracker.take_failed_exports
+            flushed = drained == SUCCESS && failed_exports.zero?
+          end
+        end
+        return false unless flush_thread.join([timeout, 0].max)
+
+        flushed
+      end
+
+      def shutdown(timeout = 30.0)
+        deadline = Otel.monotonic_now + [timeout, 0].max
+        @state_mutex.synchronize { @closed = true }
+        flushed = flush([deadline - Otel.monotonic_now, 0].max)
+
+        processor = @state_mutex.synchronize do
+          current = @processor
+          @processor = nil
+          @provider = nil
+          @delivery_tracker = nil
+          @tracer = nil
+          current
+        end
+        return finish_shutdown(flushed) if processor.nil?
+
+        shutdown_thread = Thread.new { processor.shutdown(timeout: [deadline - Otel.monotonic_now, 0].max) }
+        joined = shutdown_thread.join([deadline - Otel.monotonic_now, 0].max)
+        finish_shutdown(flushed && !joined.nil?)
+      end
+
+      private
+
+      def finish_shutdown(succeeded)
+        Otel.unregister_transport(self)
+        succeeded
+      end
+
+      def warn_closed
+        Bitfab.warn_once(
+          "otel-submit-after-shutdown",
+          "OpenTelemetry transport is shut down; dropping spans"
+        )
+        nil
+      end
+
+      # A forked child inherits the parent's provider and processor but none of
+      # its threads, so the pipeline is rebuilt on the first submit after a fork.
+      # Callers hold @state_mutex, so exactly one thread rebuilds.
+      def ensure_process
+        Otel.ensure_process_state
+        return if @owner_pid == Process.pid && !@processor.nil?
+        return if @closed
+
+        @owner_pid = Process.pid
+        create_pipeline
+        Otel.register_transport(self)
+      end
+
+      def create_pipeline
+        @delivery_tracker = DeliveryTrackingExporter.new(create_exporter)
+        @provider = OpenTelemetry::SDK::Trace::TracerProvider.new(
+          sampler: OpenTelemetry::SDK::Trace::Samplers::ALWAYS_ON,
+          resource: OpenTelemetry::SDK::Resources::Resource.create(
+            "service.name" => "bitfab-ruby-sdk",
+            "service.version" => Bitfab::VERSION
+          ),
+          span_limits: OpenTelemetry::SDK::Trace::SpanLimits.new(
+            attribute_count_limit: 2,
+            attribute_length_limit: nil,
+            event_count_limit: 128,
+            link_count_limit: 128,
+            event_attribute_count_limit: 128,
+            event_attribute_length_limit: nil,
+            link_attribute_count_limit: 128
+          )
+        )
+        @processor = OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new(
+          @delivery_tracker,
+          exporter_timeout: EXPORT_TIMEOUT_MILLIS,
+          schedule_delay: SCHEDULE_DELAY_MILLIS,
+          max_queue_size: @max_queue_size,
+          max_export_batch_size: @max_export_batch_size
+        )
+        @provider.add_span_processor(@processor)
+        @tracer = @provider.tracer("bitfab", Bitfab::VERSION)
+      end
+
+      def create_exporter
+        DirectExporter.new(
+          direct_sender: @direct_sender,
+          max_request_bytes: @max_request_bytes,
+          max_request_batch_size: @max_request_batch_size,
+          export_concurrency: @export_concurrency
+        )
+      end
+    end
+
+    # Turns finished carrier spans into the OTLP/JSON shapes Bitfab ingests.
+    module Encoder
+      class << self
+        def encode_span(span)
+          encoded = JSON.generate(span_to_otlp(span))
+          EncodedSpan.new(encoded, encoded.bytesize)
+        end
+
+        def request_envelope(first)
+          scope = first.instrumentation_scope
+          resource = JSON.generate({"attributes" => attributes(first.resource.attribute_enumerator.to_h)})
+          scope_json = JSON.generate({"name" => scope.name, "version" => scope.version || ""})
+          head = %({"resourceSpans":[{"resource":#{resource},"scopeSpans":[{"scope":#{scope_json},"spans":[)
+          tail = "]}]}]}"
+          RequestEnvelope.new(head, tail, (head + tail).bytesize)
+        end
+
+        def encode_request(envelope, spans)
+          envelope.head + spans.map(&:encoded).join(",") + envelope.tail
+        end
+
+        def span_to_otlp(span)
+          result = {
+            "traceId" => span.hex_trace_id,
+            "spanId" => span.hex_span_id,
+            "name" => span.name,
+            "kind" => SPAN_KINDS.fetch(span.kind, 1),
+            "startTimeUnixNano" => (span.start_timestamp || 0).to_s,
+            "endTimeUnixNano" => (span.end_timestamp || 0).to_s,
+            "attributes" => attributes(span.attributes),
+            "droppedAttributesCount" => dropped_count(span.total_recorded_attributes, span.attributes),
+            "droppedEventsCount" => dropped_count(span.total_recorded_events, span.events),
+            "droppedLinksCount" => dropped_count(span.total_recorded_links, span.links),
+            "status" => status(span.status),
+            "flags" => span.trace_flags.sampled? ? 1 : 0
+          }
+          result["parentSpanId"] = span.hex_parent_span_id unless span.parent_span_id == INVALID_SPAN_ID
+          tracestate = span.tracestate&.to_s
+          result["traceState"] = tracestate unless tracestate.nil? || tracestate.empty?
+          result
+        end
+
+        def span_name(operation, payload)
+          raw_span = payload["rawSpan"]
+          if operation == "external_span" && raw_span.is_a?(Hash)
+            name = raw_span.dig("span_data", "name")
+            return name if name.is_a?(String)
+          end
+          return payload["traceFunctionKey"] if payload["traceFunctionKey"].is_a?(String)
+
+          "bitfab.#{operation}"
+        end
+
+        def timestamp(payload, field)
+          raw = payload.dig("rawSpan", field) if payload["rawSpan"].is_a?(Hash)
+          if raw.nil?
+            raw_trace = payload["externalTrace"] || payload["rawTrace"]
+            raw = raw_trace[field] if raw_trace.is_a?(Hash)
+          end
+          return Time.now unless raw.is_a?(String)
+
+          begin
+            Time.iso8601(raw)
+          rescue ArgumentError
+            Time.now
+          end
+        end
+
+        def error?(payload)
+          raw_span = payload["rawSpan"]
+          return true if raw_span.is_a?(Hash) && !raw_span.dig("span_data", "error").nil?
+
+          errors = payload["errors"]
+          errors.is_a?(Array) && !errors.empty?
+        end
+
+        private
+
+        def status(span_status)
+          result = {"code" => span_status.code}
+          description = span_status.description
+          result["message"] = description unless description.nil? || description.empty?
+          result
+        end
+
+        def dropped_count(total_recorded, recorded)
+          [total_recorded - (recorded&.size || 0), 0].max
+        end
+
+        def attributes(values)
+          (values || {}).map { |key, value| {"key" => key.to_s, "value" => attribute_value(value)} }
+        end
+
+        def attribute_value(value)
+          case value
+          when true, false then {"boolValue" => value}
+          when Integer then {"intValue" => value.to_s}
+          when Float then {"doubleValue" => value}
+          when String then {"stringValue" => value}
+          when Array then {"arrayValue" => {"values" => value.map { |item| attribute_value(item) }}}
+          else {"stringValue" => value.to_s}
+          end
+        end
+      end
+    end
+  end
+end