fast-prometheus 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AGENTS.md +8 -1
- data/CHANGELOG.md +108 -0
- data/README.md +15 -8
- data/lib/fast/prometheus/counter.rb +1 -1
- data/lib/fast/prometheus/formats/metrics_proto.rb +95 -0
- data/lib/fast/prometheus/formats/protobuf.rb +209 -89
- data/lib/fast/prometheus/formats/text.rb +48 -38
- data/lib/fast/prometheus/gauge.rb +7 -3
- data/lib/fast/prometheus/histogram.rb +53 -17
- data/lib/fast/prometheus/metric.rb +76 -16
- data/lib/fast/prometheus/native_histogram.rb +22 -8
- data/lib/fast/prometheus/otlp/mapper.rb +59 -61
- data/lib/fast/prometheus/otlp/proto.rb +169 -0
- data/lib/fast/prometheus/otlp/service_interface.rb +3 -3
- data/lib/fast/prometheus/registry.rb +21 -9
- data/lib/fast/prometheus/snapshot.rb +29 -55
- data/lib/fast/prometheus/store.rb +4 -2
- data/lib/fast/prometheus/summary.rb +22 -3
- data/lib/fast/prometheus/version.rb +1 -1
- metadata +7 -10
- data/lib/fast/prometheus/formats/metrics_pb.rb +0 -33
- data/lib/fast/prometheus/otlp/pb/opentelemetry/proto/collector/metrics/v1/metrics_service_pb.rb +0 -27
- data/lib/fast/prometheus/otlp/pb/opentelemetry/proto/common/v1/common_pb.rb +0 -26
- data/lib/fast/prometheus/otlp/pb/opentelemetry/proto/metrics/v1/metrics_pb.rb +0 -41
- data/lib/fast/prometheus/otlp/pb/opentelemetry/proto/resource/v1/resource_pb.rb +0 -23
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 567abd58fbe511ba93717dbec96fa202313d689ed1fb30b765dace0de60d141f
|
|
4
|
+
data.tar.gz: 6c9cf404e4f7c0b7f54ea65f0f6c009ea488abdbf0bb2e1705d9c8e43e3b3b46
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: cc0c6d858de7e4893e84dc615694a1488a84829eefed4511ad8ca445aa2be354f54b382e3edb54c78363bb5b1b4625ae026db395471fc47a49b3e027bbaa1c7e
|
|
7
|
+
data.tar.gz: 1e0c1751a0ade41579e5932044b37d5cecd4df4324bdcccc97d4886bf62d0569e1f030312645b8872f327ce2ffda25c3c525e77a3fe23022a7de125b30f5aec3
|
data/AGENTS.md
CHANGED
|
@@ -9,9 +9,16 @@ bundle install
|
|
|
9
9
|
bundle exec sus
|
|
10
10
|
bundle exec rubocop
|
|
11
11
|
bundle exec ruby script/e2e_prometheus_scrape.rb
|
|
12
|
-
BENCH_QUICK=1 bundle exec ruby benchmark/observe.rb
|
|
12
|
+
BENCH_QUICK=1 bundle exec ruby benchmark/observe.rb # observe/get vs prometheus-client, one table
|
|
13
|
+
BENCH_QUICK=1 bundle exec ruby benchmark/exposition.rb # scrape suite: end to end, by stage, by size, vs google-protobuf
|
|
14
|
+
protoc --proto_path=proto --ruby_out=fixtures/pb proto/metrics.proto proto/opentelemetry/proto/**/*.proto # regenerate the reference decoders (tests only)
|
|
15
|
+
bundle exec ruby script/scrape_rss.rb --mode server --series 7500 # RSS per scrape, see header
|
|
13
16
|
```
|
|
14
17
|
|
|
18
|
+
## Dependencies
|
|
19
|
+
|
|
20
|
+
- `google-protobuf` is a development dependency only: tests decode the library's output with it (`fixtures/reference.rb`). Runtime protobuf is `fast-protowire`; the message declarations live in `lib/fast/prometheus/formats/metrics_proto.rb` and `lib/fast/prometheus/otlp/proto.rb` and must mirror `proto/`.
|
|
21
|
+
|
|
15
22
|
## Style Rules
|
|
16
23
|
|
|
17
24
|
- 2-space indentation
|
data/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,114 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.3.0] - 2026-09-18
|
|
11
|
+
|
|
12
|
+
### Changed
|
|
13
|
+
|
|
14
|
+
- Protobuf exposition and OTLP export no longer use `google-protobuf`. The
|
|
15
|
+
Prometheus client model and the OTLP messages are declared with
|
|
16
|
+
[fast-protowire](https://github.com/jetpks/fast-protowire), a wire-format
|
|
17
|
+
library with no native extension: no per-message arenas, no object cache,
|
|
18
|
+
no descriptor pool. Output is byte-identical (tests decode it with
|
|
19
|
+
google-protobuf as a development dependency). `Formats::Protobuf.render`
|
|
20
|
+
encodes each series on its own and appends it to the family's bytes, so a
|
|
21
|
+
36k-series scrape peaks at roughly the body size instead of ~320 MiB.
|
|
22
|
+
- `OTLP::Mapper#request` returns a `Fast::Prometheus::OTLP::Proto::
|
|
23
|
+
ExportMetricsServiceRequest`; the gRPC interface's response class is
|
|
24
|
+
`Fast::Prometheus::OTLP::Proto::ExportMetricsServiceResponse`.
|
|
25
|
+
- New `script/scrape_rss.rb`: an RSS-per-scrape harness for the exposition
|
|
26
|
+
path (forked server scraped over keep-alive, or in-process loops).
|
|
27
|
+
- `Formats::Text.render` and `Formats::Protobuf.render` allocate per sample
|
|
28
|
+
and per series, not per label: text appends every piece straight into the
|
|
29
|
+
output (one String per sample line, the value), and protobuf writes each
|
|
30
|
+
series' bytes with `Fast::Protowire::Wire` into one reused scratch buffer,
|
|
31
|
+
with no message objects. Over 5,200 series x 12 labels a text render went
|
|
32
|
+
from 100,840 to 10,631 objects and a protobuf render from 342,705 to 6,165
|
|
33
|
+
(0.114 s to 0.030 s); in the 36k-series server harness, protobuf+gzip
|
|
34
|
+
scrapes went from 0.92 s to 0.25 s and text from 0.20 s to 0.08 s.
|
|
35
|
+
- `MetricSnapshot#series` is the metric's store copied under its lock: a
|
|
36
|
+
frozen `Hash` from each series' label values (an `Array` in `label_names`
|
|
37
|
+
order) to its value, with `MetricSnapshot#labels(values)` for the
|
|
38
|
+
`{name => value}` view. The `Series` object and its per-series label Hash
|
|
39
|
+
are gone, and with them the snapshot's per-series cost: `Registry#collect`
|
|
40
|
+
over 36,000 series x 12 labels went from 34 ms, 72,056 objects and 19 MiB
|
|
41
|
+
to 1 ms, 342 objects and the 2 MiB copy. `Metric#snapshot_values` is keyed
|
|
42
|
+
the same way; `Metric#values` keeps its label-Hash keys. `HistogramValue`,
|
|
43
|
+
`SummaryValue` and `NativeHistogramValue` are frozen `Struct`s rather than
|
|
44
|
+
`Data`, since `Data.new` allocates two objects besides the instance;
|
|
45
|
+
`Histogram#cumulative_buckets` and the native histogram bucket readers
|
|
46
|
+
return frozen pairs.
|
|
47
|
+
- A bound or unlabeled `increment`/`observe`/`set` allocates nothing (the
|
|
48
|
+
`labels: {}` default is one shared frozen Hash and the store lock is taken
|
|
49
|
+
with `yield`, not a captured block); a labeled call allocates only its
|
|
50
|
+
resolved key. `test/fast/prometheus/allocations.rb` freezes these budgets.
|
|
51
|
+
- `fast-protowire` is resolved from rubygems.org (`~> 0.1`); the
|
|
52
|
+
sibling-path override is gone from `gems.rb`. Development and the
|
|
53
|
+
published benchmarks are on Ruby 4.0.7 (`mise.toml`, CI's dev row).
|
|
54
|
+
|
|
55
|
+
### Added
|
|
56
|
+
|
|
57
|
+
- `benchmark/exposition.rb`, a scrape benchmark suite: end-to-end scrapes
|
|
58
|
+
through `Middleware::Exporter` over `Async::HTTP` in every content variant,
|
|
59
|
+
each stage of a scrape on its own (collect, render, gzip) with objects and
|
|
60
|
+
malloc bytes, the renderers against `prometheus-client`'s text formatter
|
|
61
|
+
from 1k to 100k series, and the protobuf renderer against the two
|
|
62
|
+
google-protobuf encoders this gem used to ship with live native arenas and
|
|
63
|
+
GC time. `benchmark/observe.rb` reports every metric operation against
|
|
64
|
+
`prometheus-client` as one table: i/s, speedup and objects per call.
|
|
65
|
+
|
|
66
|
+
## [0.2.0] - 2026-09-15
|
|
67
|
+
|
|
68
|
+
### Added
|
|
69
|
+
|
|
70
|
+
- `Registry#exist?(name)` — true iff a metric is registered under `name`.
|
|
71
|
+
- `Metric#init_label_set(labels)` is now public: it resolves `labels` like any
|
|
72
|
+
mutator and creates that series at the type's zero value under the store
|
|
73
|
+
lock, only if it's absent — it never resets a live series.
|
|
74
|
+
- Every metric now seeds a zero-valued series at construction when it's fully
|
|
75
|
+
bound (no declared labels, or `preset_labels:` covers every declared
|
|
76
|
+
label); a fully-bound `with_labels` child seeds its own series the same
|
|
77
|
+
way. A seeded series shows up in `values`, `Registry#collect`, and text/
|
|
78
|
+
protobuf exposition at zero — for example a freshly registered counter now
|
|
79
|
+
renders `requests_total 0.0` instead of being absent from scrapes until
|
|
80
|
+
first incremented. Partially-bound metrics and zero-valued
|
|
81
|
+
`increment`/`decrement` still seed nothing; `get` still never creates a
|
|
82
|
+
series.
|
|
83
|
+
- `Gauge#set_to_current_time(labels: {})` — `set(Time.now.to_f, labels:
|
|
84
|
+
labels)`.
|
|
85
|
+
- `Metric#snapshot_values` — every series' value in its frozen snapshot shape
|
|
86
|
+
(`HistogramValue`, `SummaryValue`, `NativeHistogramValue`, or the
|
|
87
|
+
already-frozen `Float` for `Counter`/`Gauge`), keyed by label hash, built
|
|
88
|
+
under the store lock.
|
|
89
|
+
|
|
90
|
+
### Changed
|
|
91
|
+
|
|
92
|
+
- **Breaking:** `Metric` names may now be given as a `String` or a `Symbol`
|
|
93
|
+
and are normalized to a `Symbol` on construction; `Metric#name` always
|
|
94
|
+
returns a `Symbol` where it previously echoed back whatever was passed in.
|
|
95
|
+
`Registry#unregister`, `#get`, `#exist?`, and `#fetch_or_register` accept
|
|
96
|
+
either form and symbolize it before lookup; `DuplicateMetric` now fires
|
|
97
|
+
across forms (registering `"requests_total"` after `:requests_total` is
|
|
98
|
+
already registered raises).
|
|
99
|
+
- **Breaking:** `Metric#label_names` is renamed to `Metric#labels`; the old
|
|
100
|
+
name no longer exists (no alias). `MetricSnapshot#label_names` is
|
|
101
|
+
unaffected.
|
|
102
|
+
- **Breaking:** `Histogram#get`/`#values` and `Summary#get`/`#values` now
|
|
103
|
+
return prometheus-client-shaped hashes instead of the internal
|
|
104
|
+
`HistogramSlot`/`Summary::Value` storage objects. `Histogram#get` returns a
|
|
105
|
+
fresh `Hash` keyed by each bucket boundary's `to_s` (ascending), then
|
|
106
|
+
`"+Inf"`, then `"sum"`, with cumulative `Integer` bucket counts and a
|
|
107
|
+
`Float` sum; `Summary#get` returns `{ "count" => Integer, "sum" => Float }`.
|
|
108
|
+
`Histogram#cumulative_buckets`, `#sum`, and `#count` now return zero-valued
|
|
109
|
+
results for an unobserved series instead of `nil`. `NativeHistogram#get`/
|
|
110
|
+
`#values` now return the frozen `NativeHistogramValue` snapshot type
|
|
111
|
+
(previously the internal `NativeHistogram::Slot`), including a zero-valued
|
|
112
|
+
value for an unobserved series. All of these are fresh copies per call;
|
|
113
|
+
mutating a returned `Hash` no longer affects the metric.
|
|
114
|
+
- `Fast::Prometheus.registry=` now assigns under the same lock the reader
|
|
115
|
+
uses, closing a race between a writer and a concurrent reader/memoizing
|
|
116
|
+
first read.
|
|
117
|
+
|
|
10
118
|
## [0.1.0] - 2026-09-11
|
|
11
119
|
|
|
12
120
|
### Added
|
data/README.md
CHANGED
|
@@ -21,7 +21,7 @@ Or add to your `Gemfile`:
|
|
|
21
21
|
gem "fast-prometheus"
|
|
22
22
|
```
|
|
23
23
|
|
|
24
|
-
`require "fast/prometheus"` loads only the core, with no IO dependencies. The
|
|
24
|
+
`require "fast/prometheus"` loads only the core, with no IO dependencies. Protobuf exposition and OTLP export are built on [fast-protowire](https://github.com/jetpks/fast-protowire) rather than `google-protobuf`, so no native protobuf runtime is loaded either. The
|
|
25
25
|
opt-in surfaces and what each pulls in are listed in
|
|
26
26
|
[the require-path reference](docs/reference/require-paths.md).
|
|
27
27
|
|
|
@@ -76,7 +76,7 @@ under Falcon and scraping it with a real Prometheus, see
|
|
|
76
76
|
- [Concurrency model](docs/explanation/concurrency.md) — the locking contract and what it guarantees under threads and fibers.
|
|
77
77
|
- [Native histograms](docs/explanation/native-histograms.md) — what a native histogram is and why it's protobuf-only.
|
|
78
78
|
- [Design: why a new gem](docs/explanation/design.md) — why fast-prometheus exists instead of a `client_ruby` fork.
|
|
79
|
-
- [Benchmarks](docs/explanation/benchmarks.md) —
|
|
79
|
+
- [Benchmarks](docs/explanation/benchmarks.md) — scrape and observe benchmarks against prometheus-client and google-protobuf, and how to reproduce them.
|
|
80
80
|
|
|
81
81
|
## Concurrency
|
|
82
82
|
|
|
@@ -90,13 +90,18 @@ handled it; aggregating across processes is out of scope. See
|
|
|
90
90
|
|
|
91
91
|
## Performance
|
|
92
92
|
|
|
93
|
-
Single process, locked, thread-safe by default (`benchmark-ips`
|
|
94
|
-
against `prometheus-client`; see
|
|
95
|
-
full
|
|
93
|
+
Single process, locked, thread-safe by default (`benchmark-ips` and allocation
|
|
94
|
+
comparisons against `prometheus-client` and `google-protobuf`; see
|
|
95
|
+
[the benchmarks page](docs/explanation/benchmarks.md) for the full runs and conditions):
|
|
96
96
|
|
|
97
|
-
-
|
|
98
|
-
-
|
|
99
|
-
|
|
97
|
+
- A protobuf scrape of 36,000 series renders in **29 objects**, where the
|
|
98
|
+
`google-protobuf` encoder needed 3.5 million Ruby objects and 504k native arenas, and
|
|
99
|
+
in a hundredth of the GC time; taking the snapshot it reads is 1 ms and one Hash per metric
|
|
100
|
+
- The text renderer allocates **66x fewer objects** than prometheus-client's formatter
|
|
101
|
+
for the same body, and renders it 3.2x faster at 36,000 series
|
|
102
|
+
- Bound counter and gauge writes allocate nothing and are **1.45x** faster than
|
|
103
|
+
prometheus-client's; histogram observe is **3.1x** faster and summary observe **3.3x**
|
|
104
|
+
- Native histogram observe runs at **1.32M i/s** with no prometheus-client equivalent
|
|
100
105
|
|
|
101
106
|
## Development
|
|
102
107
|
|
|
@@ -104,6 +109,8 @@ full run and conditions):
|
|
|
104
109
|
bundle exec sus
|
|
105
110
|
bundle exec rubocop
|
|
106
111
|
E2E_REQUIRED=1 ./integration/run
|
|
112
|
+
BENCH_QUICK=1 bundle exec ruby benchmark/observe.rb # every metric operation vs prometheus-client
|
|
113
|
+
BENCH_QUICK=1 bundle exec ruby benchmark/exposition.rb # the scrape suite
|
|
107
114
|
```
|
|
108
115
|
|
|
109
116
|
See [how to verify a release build](docs/how-to/verify-a-release.md) for the pre-release rule.
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fast/protowire"
|
|
4
|
+
|
|
5
|
+
module Fast
|
|
6
|
+
module Prometheus
|
|
7
|
+
module Formats
|
|
8
|
+
module Protobuf
|
|
9
|
+
# The Prometheus client data model (proto/metrics.proto,
|
|
10
|
+
# io.prometheus.client) declared for the wire. The checked-in proto
|
|
11
|
+
# is proto3, so scalars have implicit presence and a zero value is
|
|
12
|
+
# omitted. Only the fields the exposition emits or a scraper reads
|
|
13
|
+
# back are declared; exemplars and created timestamps ride along as
|
|
14
|
+
# unknown fields.
|
|
15
|
+
module Proto
|
|
16
|
+
MetricType = Fast::Protowire::Enum.define(COUNTER: 0, GAUGE: 1, SUMMARY: 2, UNTYPED: 3, HISTOGRAM: 4,
|
|
17
|
+
GAUGE_HISTOGRAM: 5)
|
|
18
|
+
|
|
19
|
+
class LabelPair < Fast::Protowire::Message
|
|
20
|
+
field :name, :string, 1
|
|
21
|
+
field :value, :string, 2
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
class Gauge < Fast::Protowire::Message
|
|
25
|
+
field :value, :double, 1
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
class Counter < Fast::Protowire::Message
|
|
29
|
+
field :value, :double, 1
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
class Quantile < Fast::Protowire::Message
|
|
33
|
+
field :quantile, :double, 1
|
|
34
|
+
field :value, :double, 2
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
class Summary < Fast::Protowire::Message
|
|
38
|
+
field :sample_count, :uint64, 1
|
|
39
|
+
field :sample_sum, :double, 2
|
|
40
|
+
repeated :quantile, Quantile, 3
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
class Untyped < Fast::Protowire::Message
|
|
44
|
+
field :value, :double, 1
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
class Bucket < Fast::Protowire::Message
|
|
48
|
+
field :cumulative_count, :uint64, 1
|
|
49
|
+
field :upper_bound, :double, 2
|
|
50
|
+
field :cumulative_count_float, :double, 4
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
class BucketSpan < Fast::Protowire::Message
|
|
54
|
+
field :offset, :sint32, 1
|
|
55
|
+
field :length, :uint32, 2
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
class Histogram < Fast::Protowire::Message
|
|
59
|
+
field :sample_count, :uint64, 1
|
|
60
|
+
field :sample_sum, :double, 2
|
|
61
|
+
repeated :bucket, Bucket, 3
|
|
62
|
+
field :sample_count_float, :double, 4
|
|
63
|
+
field :schema, :sint32, 5
|
|
64
|
+
field :zero_threshold, :double, 6
|
|
65
|
+
field :zero_count, :uint64, 7
|
|
66
|
+
field :zero_count_float, :double, 8
|
|
67
|
+
repeated :negative_span, BucketSpan, 9
|
|
68
|
+
repeated :negative_delta, :sint64, 10
|
|
69
|
+
repeated :negative_count, :double, 11
|
|
70
|
+
repeated :positive_span, BucketSpan, 12
|
|
71
|
+
repeated :positive_delta, :sint64, 13
|
|
72
|
+
repeated :positive_count, :double, 14
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
class Metric < Fast::Protowire::Message
|
|
76
|
+
repeated :label, LabelPair, 1
|
|
77
|
+
field :gauge, Gauge, 2
|
|
78
|
+
field :counter, Counter, 3
|
|
79
|
+
field :summary, Summary, 4
|
|
80
|
+
field :untyped, Untyped, 5
|
|
81
|
+
field :timestamp_ms, :int64, 6
|
|
82
|
+
field :histogram, Histogram, 7
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
class MetricFamily < Fast::Protowire::Message
|
|
86
|
+
field :name, :string, 1
|
|
87
|
+
field :help, :string, 2
|
|
88
|
+
field :type, MetricType, 3
|
|
89
|
+
repeated :metric, Metric, 4
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|
|
@@ -1,97 +1,234 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "fast/prometheus"
|
|
4
|
-
require_relative "
|
|
4
|
+
require_relative "metrics_proto"
|
|
5
5
|
|
|
6
6
|
module Fast
|
|
7
7
|
module Prometheus
|
|
8
8
|
module Formats
|
|
9
|
+
# Prometheus protobuf exposition: a stream of varint-length-prefixed
|
|
10
|
+
# MetricFamily messages.
|
|
11
|
+
#
|
|
12
|
+
# Each family's header is a declared Proto::MetricFamily; the series
|
|
13
|
+
# under it are written straight to the wire with Fast::Protowire::Wire,
|
|
14
|
+
# no LabelPair/Metric/Counter objects, because a scrape's series count
|
|
15
|
+
# is the one thing here that is large. A series' bytes go into one
|
|
16
|
+
# scratch buffer reused for the whole render and are copied under the
|
|
17
|
+
# family's metric tag, so rendering allocates per series only what
|
|
18
|
+
# Array#pack needs for a double. Native histograms, few and irregular,
|
|
19
|
+
# still go through Proto::Histogram. Bytes are identical to what the
|
|
20
|
+
# declared classes (and google-protobuf) produce; the tests check.
|
|
9
21
|
module Protobuf
|
|
10
22
|
CONTENT_TYPE = "application/vnd.google.protobuf; proto=io.prometheus.client.MetricFamily; encoding=delimited"
|
|
11
23
|
|
|
12
24
|
TYPE_MAP = {
|
|
13
|
-
counter:
|
|
14
|
-
gauge:
|
|
15
|
-
summary:
|
|
16
|
-
histogram:
|
|
17
|
-
native_histogram:
|
|
25
|
+
counter: :COUNTER,
|
|
26
|
+
gauge: :GAUGE,
|
|
27
|
+
summary: :SUMMARY,
|
|
28
|
+
histogram: :HISTOGRAM,
|
|
29
|
+
native_histogram: :HISTOGRAM
|
|
18
30
|
}.freeze
|
|
19
31
|
private_constant :TYPE_MAP
|
|
20
32
|
|
|
33
|
+
# Field tags for the messages written by hand, read off the
|
|
34
|
+
# declarations so field numbers live in metrics_proto.rb only. All
|
|
35
|
+
# are one byte (field numbers below 16), which the size arithmetic
|
|
36
|
+
# below relies on.
|
|
37
|
+
def self.tag(message_class, field)
|
|
38
|
+
field = message_class.fields.fetch(field)
|
|
39
|
+
Fast::Protowire::Wire.tag(field.number, field.wire_type)
|
|
40
|
+
end
|
|
41
|
+
private_class_method :tag
|
|
42
|
+
|
|
43
|
+
FAMILY_METRIC = tag(Proto::MetricFamily, :metric)
|
|
44
|
+
METRIC_LABEL = tag(Proto::Metric, :label)
|
|
45
|
+
METRIC_GAUGE = tag(Proto::Metric, :gauge)
|
|
46
|
+
METRIC_COUNTER = tag(Proto::Metric, :counter)
|
|
47
|
+
METRIC_SUMMARY = tag(Proto::Metric, :summary)
|
|
48
|
+
METRIC_HISTOGRAM = tag(Proto::Metric, :histogram)
|
|
49
|
+
LABEL_NAME = tag(Proto::LabelPair, :name)
|
|
50
|
+
LABEL_VALUE = tag(Proto::LabelPair, :value)
|
|
51
|
+
VALUE = tag(Proto::Counter, :value) # Gauge.value has the same tag
|
|
52
|
+
SUMMARY_COUNT = tag(Proto::Summary, :sample_count)
|
|
53
|
+
SUMMARY_SUM = tag(Proto::Summary, :sample_sum)
|
|
54
|
+
HISTOGRAM_COUNT = tag(Proto::Histogram, :sample_count)
|
|
55
|
+
HISTOGRAM_SUM = tag(Proto::Histogram, :sample_sum)
|
|
56
|
+
HISTOGRAM_BUCKET = tag(Proto::Histogram, :bucket)
|
|
57
|
+
BUCKET_COUNT = tag(Proto::Bucket, :cumulative_count)
|
|
58
|
+
BUCKET_BOUND = tag(Proto::Bucket, :upper_bound)
|
|
59
|
+
private_constant :FAMILY_METRIC, :METRIC_LABEL, :METRIC_GAUGE, :METRIC_COUNTER, :METRIC_SUMMARY,
|
|
60
|
+
:METRIC_HISTOGRAM, :LABEL_NAME, :LABEL_VALUE, :VALUE, :SUMMARY_COUNT, :SUMMARY_SUM,
|
|
61
|
+
:HISTOGRAM_COUNT, :HISTOGRAM_SUM, :HISTOGRAM_BUCKET, :BUCKET_COUNT, :BUCKET_BOUND
|
|
62
|
+
|
|
63
|
+
# Tag plus eight bytes: what a double field occupies when sent.
|
|
64
|
+
DOUBLE_SIZE = 9
|
|
65
|
+
private_constant :DOUBLE_SIZE
|
|
66
|
+
|
|
67
|
+
# One varint-length-prefixed MetricFamily frame per metric.
|
|
21
68
|
def self.render(snapshot)
|
|
69
|
+
scratch = String.new
|
|
22
70
|
snapshot.metrics.each_with_object(String.new) do |ms, buffer|
|
|
23
|
-
|
|
71
|
+
family = encode_metric_family(ms, scratch)
|
|
72
|
+
Fast::Protowire::Wire.append_varint(buffer, family.bytesize)
|
|
73
|
+
buffer << family
|
|
24
74
|
end
|
|
25
75
|
end
|
|
26
76
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
77
|
+
# A family's bytes are its header (name, help, type) followed by one
|
|
78
|
+
# metric entry per series, each written into +scratch+ and copied.
|
|
79
|
+
private_class_method def self.encode_metric_family(metric_snapshot, scratch)
|
|
80
|
+
header = Proto::MetricFamily.new(name: metric_snapshot.name.name, help: metric_snapshot.docstring,
|
|
81
|
+
type: TYPE_MAP.fetch(metric_snapshot.type))
|
|
82
|
+
type = metric_snapshot.type
|
|
83
|
+
names = metric_snapshot.label_names
|
|
84
|
+
buffer = header.encode
|
|
85
|
+
metric_snapshot.series.each_pair do |values, value|
|
|
86
|
+
scratch.clear
|
|
87
|
+
append_metric(scratch, names, values, value, type)
|
|
88
|
+
buffer << FAMILY_METRIC
|
|
89
|
+
Fast::Protowire::Wire.append_varint(buffer, scratch.bytesize)
|
|
90
|
+
buffer << scratch
|
|
39
91
|
end
|
|
92
|
+
buffer
|
|
93
|
+
end
|
|
40
94
|
|
|
95
|
+
# Metric: labels (1) then the one value field for the type, in
|
|
96
|
+
# field-number order as the declared encoder would write them.
|
|
97
|
+
private_class_method def self.append_metric(out, names, values, value, type)
|
|
98
|
+
names.each_index { |i| append_label_pair(out, names[i], values[i]) }
|
|
41
99
|
case type
|
|
42
|
-
when :counter
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
)
|
|
47
|
-
when :gauge
|
|
48
|
-
Io::Prometheus::Client::Metric.new(
|
|
49
|
-
label: labels,
|
|
50
|
-
gauge: Io::Prometheus::Client::Gauge.new(value: series.value)
|
|
51
|
-
)
|
|
52
|
-
when :summary
|
|
53
|
-
Io::Prometheus::Client::Metric.new(
|
|
54
|
-
label: labels,
|
|
55
|
-
summary: Io::Prometheus::Client::Summary.new(
|
|
56
|
-
sample_count: series.value.count,
|
|
57
|
-
sample_sum: series.value.sum
|
|
58
|
-
)
|
|
59
|
-
)
|
|
60
|
-
when :histogram
|
|
61
|
-
buckets = series.value.cumulative_buckets.map do |bound, count|
|
|
62
|
-
Io::Prometheus::Client::Bucket.new(
|
|
63
|
-
upper_bound: bound,
|
|
64
|
-
cumulative_count: count
|
|
65
|
-
)
|
|
66
|
-
end
|
|
67
|
-
Io::Prometheus::Client::Metric.new(
|
|
68
|
-
label: labels,
|
|
69
|
-
histogram: Io::Prometheus::Client::Histogram.new(
|
|
70
|
-
sample_count: series.value.count,
|
|
71
|
-
sample_sum: series.value.sum,
|
|
72
|
-
bucket: buckets
|
|
73
|
-
)
|
|
74
|
-
)
|
|
100
|
+
when :counter then append_scalar_message(out, METRIC_COUNTER, value)
|
|
101
|
+
when :gauge then append_scalar_message(out, METRIC_GAUGE, value)
|
|
102
|
+
when :summary then append_summary(out, value)
|
|
103
|
+
when :histogram then append_histogram(out, value)
|
|
75
104
|
when :native_histogram
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
105
|
+
Fast::Protowire::Wire.append_length_delimited(out, METRIC_HISTOGRAM, native_histogram(value).encode)
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# LabelPair { string name = 1; string value = 2 }.
|
|
110
|
+
private_class_method def self.append_label_pair(out, name, value)
|
|
111
|
+
name = name.name if name.is_a?(Symbol)
|
|
112
|
+
value = value.to_s
|
|
113
|
+
name_size = name.bytesize
|
|
114
|
+
value_size = value.bytesize
|
|
115
|
+
out << METRIC_LABEL
|
|
116
|
+
append_varint(out, 2 + varint_size(name_size) + name_size + varint_size(value_size) + value_size)
|
|
117
|
+
Fast::Protowire::Wire.append_length_delimited(out, LABEL_NAME, name)
|
|
118
|
+
Fast::Protowire::Wire.append_length_delimited(out, LABEL_VALUE, value)
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# Counter / Gauge { double value = 1 }: empty when the value is the
|
|
122
|
+
# proto3 default (+0.0 exactly; -0.0 and NaN are sent).
|
|
123
|
+
private_class_method def self.append_scalar_message(out, tag, value)
|
|
124
|
+
out << tag
|
|
125
|
+
if omit_double?(value)
|
|
126
|
+
out << 0
|
|
127
|
+
else
|
|
128
|
+
out << DOUBLE_SIZE
|
|
129
|
+
append_double(out, VALUE, value)
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# Summary { uint64 sample_count = 1; double sample_sum = 2 }.
|
|
134
|
+
private_class_method def self.append_summary(out, value)
|
|
135
|
+
count = value.count
|
|
136
|
+
sum = value.sum
|
|
137
|
+
out << METRIC_SUMMARY
|
|
138
|
+
append_varint(out, uint_size(SUMMARY_COUNT, count) + double_size(sum))
|
|
139
|
+
append_uint(out, SUMMARY_COUNT, count)
|
|
140
|
+
append_double(out, SUMMARY_SUM, sum) unless omit_double?(sum)
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# Histogram { uint64 sample_count = 1; double sample_sum = 2;
|
|
144
|
+
# repeated Bucket bucket = 3 } with Bucket { uint64 cumulative_count
|
|
145
|
+
# = 1; double upper_bound = 2 }. Sizes are summed first, then the
|
|
146
|
+
# bytes are written, so no bucket is encoded into its own buffer.
|
|
147
|
+
private_class_method def self.append_histogram(out, value)
|
|
148
|
+
count = value.count
|
|
149
|
+
sum = value.sum
|
|
150
|
+
buckets = value.cumulative_buckets
|
|
151
|
+
size = uint_size(HISTOGRAM_COUNT, count) + double_size(sum)
|
|
152
|
+
buckets.each do |bound, cumulative|
|
|
153
|
+
bucket_size = bucket_size(bound, cumulative)
|
|
154
|
+
size += 1 + varint_size(bucket_size) + bucket_size
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
out << METRIC_HISTOGRAM
|
|
158
|
+
append_varint(out, size)
|
|
159
|
+
append_uint(out, HISTOGRAM_COUNT, count)
|
|
160
|
+
append_double(out, HISTOGRAM_SUM, sum) unless omit_double?(sum)
|
|
161
|
+
buckets.each do |bound, cumulative|
|
|
162
|
+
out << HISTOGRAM_BUCKET
|
|
163
|
+
append_varint(out, bucket_size(bound, cumulative))
|
|
164
|
+
append_uint(out, BUCKET_COUNT, cumulative)
|
|
165
|
+
append_double(out, BUCKET_BOUND, bound) unless omit_double?(bound)
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
private_class_method def self.bucket_size(bound, cumulative)
|
|
170
|
+
uint_size(BUCKET_COUNT, cumulative) + double_size(bound)
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
# -- wire pieces ------------------------------------------------------
|
|
174
|
+
|
|
175
|
+
private_class_method def self.append_varint(out, value)
|
|
176
|
+
Fast::Protowire::Wire.append_varint(out, value)
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
# A uint64 field, omitted at zero.
|
|
180
|
+
private_class_method def self.append_uint(out, tag, value)
|
|
181
|
+
return if value.zero?
|
|
182
|
+
|
|
183
|
+
out << tag
|
|
184
|
+
append_varint(out, value)
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
private_class_method def self.append_double(out, tag, value)
|
|
188
|
+
out << tag
|
|
189
|
+
[value].pack("E", buffer: out)
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
private_class_method def self.uint_size(_tag, value)
|
|
193
|
+
value.zero? ? 0 : 1 + varint_size(value)
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
private_class_method def self.double_size(value)
|
|
197
|
+
omit_double?(value) ? 0 : DOUBLE_SIZE
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
# Bytes a non-negative varint takes.
|
|
201
|
+
private_class_method def self.varint_size(value)
|
|
202
|
+
size = 1
|
|
203
|
+
while value > 0x7f
|
|
204
|
+
value >>= 7
|
|
205
|
+
size += 1
|
|
94
206
|
end
|
|
207
|
+
size
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
# Implicit presence: +0.0 is not sent; -0.0 (bitwise different) is.
|
|
211
|
+
private_class_method def self.omit_double?(value)
|
|
212
|
+
value.zero? && (1.0 / value).positive?
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
# -- native histograms, through the declared classes ------------------
|
|
216
|
+
|
|
217
|
+
private_class_method def self.native_histogram(value)
|
|
218
|
+
pos_spans, pos_deltas = build_spans_deltas(value.positive_buckets)
|
|
219
|
+
neg_spans, neg_deltas = build_spans_deltas(value.negative_buckets)
|
|
220
|
+
|
|
221
|
+
Proto::Histogram.new(
|
|
222
|
+
sample_count: value.count,
|
|
223
|
+
sample_sum: value.sum,
|
|
224
|
+
schema: value.schema,
|
|
225
|
+
zero_threshold: value.zero_threshold,
|
|
226
|
+
zero_count: value.zero_count,
|
|
227
|
+
positive_span: pos_spans,
|
|
228
|
+
positive_delta: pos_deltas,
|
|
229
|
+
negative_span: neg_spans,
|
|
230
|
+
negative_delta: neg_deltas
|
|
231
|
+
)
|
|
95
232
|
end
|
|
96
233
|
|
|
97
234
|
# Build BucketSpan and sint64 delta arrays from ascending [index, count] pairs.
|
|
@@ -118,7 +255,7 @@ module Fast
|
|
|
118
255
|
if gap == 1
|
|
119
256
|
span_length += 1
|
|
120
257
|
else
|
|
121
|
-
spans <<
|
|
258
|
+
spans << Proto::BucketSpan.new(offset: span_offset, length: span_length)
|
|
122
259
|
span_offset = gap - 1
|
|
123
260
|
span_length = 1
|
|
124
261
|
end
|
|
@@ -128,26 +265,9 @@ module Fast
|
|
|
128
265
|
prev_count = count
|
|
129
266
|
end
|
|
130
267
|
|
|
131
|
-
spans <<
|
|
268
|
+
spans << Proto::BucketSpan.new(offset: span_offset, length: span_length)
|
|
132
269
|
[spans, deltas]
|
|
133
270
|
end
|
|
134
|
-
|
|
135
|
-
private_class_method def self.encode_frame(message)
|
|
136
|
-
encoded = message.class.encode(message)
|
|
137
|
-
encode_varint(encoded.bytesize) << encoded
|
|
138
|
-
end
|
|
139
|
-
|
|
140
|
-
private_class_method def self.encode_varint(value)
|
|
141
|
-
buffer = String.new
|
|
142
|
-
loop do
|
|
143
|
-
byte = value & 0x7f
|
|
144
|
-
value >>= 7
|
|
145
|
-
byte |= 0x80 if value.positive?
|
|
146
|
-
buffer << byte
|
|
147
|
-
break if value.zero?
|
|
148
|
-
end
|
|
149
|
-
buffer
|
|
150
|
-
end
|
|
151
271
|
end
|
|
152
272
|
end
|
|
153
273
|
end
|