prometheus-client 0.9.0 → 0.10.0.pre.alpha.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2ea7f2e1928d0427b0bd912c678fd6d7af8a2f44392a890ef380a5fd6a3ad62c
-  data.tar.gz: 0beea69fd3c660057752a34d2ce800ec5393e048363c216bf3921c39674937df
+  metadata.gz: 58aea5e1eb91b3b53326f8c183c65187d5fcd6d87fc908c107f73d65257809aa
+  data.tar.gz: 8d78ea6b70ebc78ed1deba5229b5e2f8b608b11511d8a4278be794dadf945284
 SHA512:
-  metadata.gz: 35e29a1a93117b8047b84ca87aa6888f9d68a53d27930603d723499b65677d497c9cd3b0831e07eda60dde55651d9a796745334e5744aff135474da1ff4cec50
-  data.tar.gz: 8291c329413756814441648f95261b904e81c856e6436694f7199269751cf047543c514e2d23940d05e42782d7baa41c6a3341ddd45d1f2db13a29aee0a22a53
+  metadata.gz: 07573f45a2555b063ad4987bd26640bfe7388f98e3f1d11e1b21c65ecf7a853aa372c32326fe944ae656efb9d6837171039ee465712f67dc50513c0f62c0baed
+  data.tar.gz: ee75b0ad0b4b2664d3d54bb3bcb29f6fe26d4e6b3801c70db37148343c73cc53b769125ad8fb56ad820d24355a2c862fe3e70832961ee7486a2b748357d205cb

data/README.md

@@ -19,12 +19,12 @@ require 'prometheus/client'
 prometheus = Prometheus::Client.registry
 
 # create a new counter metric
-http_requests = Prometheus::Client::Counter.new(:http_requests, 'A counter of HTTP requests made')
+http_requests = Prometheus::Client::Counter.new(:http_requests, docstring: 'A counter of HTTP requests made')
 # register the metric
 prometheus.register(http_requests)
 
 # equivalent helper function
-http_requests = prometheus.counter(:http_requests, 'A counter of HTTP requests made')
+http_requests = prometheus.counter(:http_requests, docstring: 'A counter of HTTP requests made')
 
 # start using the counter
 http_requests.increment
@@ -99,16 +99,16 @@ The following metric types are currently supported.
 Counter is a metric that exposes merely a sum or tally of things.
 
 ```ruby
-counter = Prometheus::Client::Counter.new(:service_requests_total, '...')
+counter = Prometheus::Client::Counter.new(:service_requests_total, docstring: '...', labels: [:service])
 
 # increment the counter for a given label set
-counter.increment({ service: 'foo' })
+counter.increment(labels: { service: 'foo' })
 
 # increment by a given value
-counter.increment({ service: 'bar' }, 5)
+counter.increment(by: 5, labels: { service: 'bar' })
 
 # get current value for a given label set
-counter.get({ service: 'bar' })
+counter.get(labels: { service: 'bar' })
 # => 5
 ```
 
@@ -118,21 +118,21 @@ Gauge is a metric that exposes merely an instantaneous value or some snapshot
 thereof.
 
 ```ruby
-gauge = Prometheus::Client::Gauge.new(:room_temperature_celsius, '...')
+gauge = Prometheus::Client::Gauge.new(:room_temperature_celsius, docstring: '...', labels: [:room])
 
 # set a value
-gauge.set({ room: 'kitchen' }, 21.534)
+gauge.set(21.534, labels: { room: 'kitchen' })
 
 # retrieve the current value for a given label set
-gauge.get({ room: 'kitchen' })
+gauge.get(labels: { room: 'kitchen' })
 # => 21.534
 
 # increment the value (default is 1)
-gauge.increment({ room: 'kitchen' })
+gauge.increment(labels: { room: 'kitchen' })
 # => 22.534
 
 # decrement the value by a given value
-gauge.decrement({ room: 'kitchen' }, 5)
+gauge.decrement(by: 5, labels: { room: 'kitchen' })
 # => 17.534
 ```
 
@@ -143,13 +143,13 @@ response sizes) and counts them in configurable buckets. It also provides a sum
 of all observed values.
 
 ```ruby
-histogram = Prometheus::Client::Histogram.new(:service_latency_seconds, '...')
+histogram = Prometheus::Client::Histogram.new(:service_latency_seconds, docstring: '...', labels: [:service])
 
 # record a value
-histogram.observe({ service: 'users' }, Benchmark.realtime { service.call(arg) })
+histogram.observe(Benchmark.realtime { service.call(arg) }, labels: { service: 'users' })
 
 # retrieve the current bucket values
-histogram.get({ service: 'users' })
+histogram.get(labels: { service: 'users' })
 # => { 0.005 => 3, 0.01 => 15, 0.025 => 18, ..., 2.5 => 42, 5 => 42, 10 = >42 }
 ```
 
@@ -158,17 +158,228 @@ histogram.get({ service: 'users' })
 Summary, similar to histograms, is an accumulator for samples. It captures
 Numeric data and provides an efficient percentile calculation mechanism.
 
+For now, only `sum` and `total` (count of observations) are supported, no actual quantiles.
+
 ```ruby
-summary = Prometheus::Client::Summary.new(:service_latency_seconds, '...')
+summary = Prometheus::Client::Summary.new(:service_latency_seconds, docstring: '...', labels: [:service])
 
 # record a value
-summary.observe({ service: 'database' }, Benchmark.realtime { service.call() })
+summary.observe(Benchmark.realtime { service.call() }, labels: { service: 'database' })
+
+# retrieve the current sum and total values
+summary_value = summary.get(labels: { service: 'database' })
+summary_value.sum # => 123.45
+summary_value.count # => 100
+```
+
+## Labels
+
+All metrics can have labels, allowing grouping of related time series.
+
+Labels are an extremely powerful feature, but one that must be used with care.
+Refer to the best practices on [naming](https://prometheus.io/docs/practices/naming/) and 
+[labels](https://prometheus.io/docs/practices/instrumentation/#use-labels).
+
+Most importantly, avoid labels that can have a large number of possible values (high 
+cardinality). For example, an HTTP Status Code is a good label. A User ID is **not**.
+
+Labels are specified optionally when updating metrics, as a hash of `label_name => value`.
+Refer to [the Prometheus documentation](https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels) 
+as to what's a valid `label_name`.
+
+In order for a metric to accept labels, their names must be specified when first initializing 
+the metric. Then, when the metric is updated, all the specified labels must be present.
+
+Example:
+
+```ruby
+https_requests_total = Counter.new(:http_requests_total, docstring: '...', labels: [:service, :status_code])
 
-# retrieve the current quantile values
-summary.get({ service: 'database' })
-# => { 0.5 => 0.1233122, 0.9 => 3.4323, 0.99 => 5.3428231 }
+# increment the counter for a given label set
+https_requests_total.increment(labels: { service: "my_service", status_code: response.status_code })
+```
+
+### Pre-set Label Values
+
+You can also "pre-set" some of these label values, if they'll always be the same, so you don't
+need to specify them every time:
+
+```ruby
+https_requests_total = Counter.new(:http_requests_total, 
+                                   docstring: '...', 
+                                   labels: [:service, :status_code],
+                                   preset_labels: { service: "my_service" })
+
+# increment the counter for a given label set
+https_requests_total.increment(labels: { status_code: response.status_code })
 ```
 
+### `with_labels`
+
+Similar to pre-setting labels, you can get a new instance of an existing metric object,
+with a subset (or full set) of labels set, so that you can increment / observe the metric
+without having to specify the labels for every call.
+
+Moreover, if all the labels the metric can take have been pre-set, validation of the labels
+is done on the call to `with_labels`, and then skipped for each observation, which can 
+lead to performance improvements. If you are incrementing a counter in a fast loop, you
+definitely want to be doing this.
+
+
+Examples:
+
+**Pre-setting labels for ease of use:**
+
+```ruby
+# in the metric definition:
+records_processed_total = registry.counter.new(:records_processed_total, 
+                                               docstring: '...', 
+                                               labels: [:service, :component],
+                                               preset_labels: { service: "my_service" })
+
+# in one-off calls, you'd specify the missing labels (component in this case)
+records_processed_total.increment(labels: { component: 'a_component' })
+
+# you can also have a "view" on this metric for a specific component where this label is
+# pre-set:
+class MyComponent
+  def metric
+    @metric ||= records_processed_total.with_labels(component: "my_component")
+  end
+  
+  def process
+    records.each do |record|
+      # process the record
+      metric.increment 
+    end
+  end
+end
+```
+
+
+## Data Stores
+
+The data for all the metrics (the internal counters associated with each labelset) 
+is stored in a global Data Store object, rather than in the metric objects themselves.
+(This "storage" is ephemeral, generally in-memory, it's not "long-term storage")
+
+The main reason to do this is that different applications may have different requirements
+for their metrics storage. Application running in pre-fork servers (like Unicorn, for
+example), require a shared store between all the processes, to be able to report coherent
+numbers. At the same time, other applications may not have this requirement but be very
+sensitive to performance, and would prefer instead a simpler, faster store.
+
+By having a standardized and simple interface that metrics use to access this store, 
+we abstract away the details of storing the data from the specific needs of each metric.
+This allows us to then simply swap around the stores based on the needs of different 
+applications, with no changes to the rest of the client. 
+
+The client provides 3 built-in stores, but if neither of these is ideal for your 
+requirements, you can easily make your own store and use that instead. More on this below.
+
+### Configuring which store to use.
+
+By default, the Client uses the `Synchronized` store, which is a simple, thread-safe Store
+for single-process scenarios.
+
+If you need to use a different store, set it in the Client Config:
+
+```ruby
+Prometheus::Client.config.data_store = Prometheus::Client::DataStores::DataStore.new(store_specific_params)
+```
+
+NOTE: You **must** make sure to set the `data_store` before initializing any metrics.
+If using Rails, you probably want to set up your Data Store on `config/application.rb`,
+or `config/environments/*`, both of which run before `config/initializers/*`
+
+Also note that `config.data_store` is set to an *instance* of a `DataStore`, not to the 
+class. This is so that the stores can receive parameters. Most of the built-in stores
+don't require any, but `DirectFileStore` does, for example.
+
+When instantiating metrics, there is an optional `store_settings` attribute. This is used
+to set up store-specific settings for each metric. For most stores, this is not used, but
+for multi-process stores, this is used to specify how to aggregate the values of each
+metric across multiple processes. For the most part, this is used for Gauges, to specify
+whether you want to report the `SUM`, `MAX` or `MIN` value observed across all processes.
+For almost all other cases, you'd leave the default (`SUM`). More on this on the 
+*Aggregation* section below.
+
+Other custom stores may also accept extra parameters besides `:aggregation`. See the
+documentation of each store for more details.
+
+### Built-in stores
+
+There are 3 built-in stores, with different trade-offs:
+
+- **Synchronized**: Default store. Thread safe, but not suitable for multi-process 
+  scenarios (e.g. pre-fork servers, like Unicorn). Stores data in Hashes, with all accesses
+  protected by Mutexes. 
+- **SingleThreaded**: Fastest store, but only suitable for single-threaded scenarios.
+  This store does not make any effort to synchronize access to its internal hashes, so 
+  it's absolutely not thread safe.
+- **DirectFileStore**: Stores data in binary files, one file per process and per metric.
+  This is generally the recommended store to use with pre-fork servers and other 
+  "multi-process" scenarios.
+
+  Each metric gets a file for each process, and manages its contents by storing keys and
+  binary floats next to them, and updating the offsets of those Floats directly. When 
+  exporting metrics, it will find all the files that apply to each metric, read them, 
+  and aggregate them.
+
+  In order to do this, each Metric needs an `:aggregation` setting, specifying how
+  to aggregate the multiple possible values we can get for each labelset. By default,
+  they are `SUM`med, which is what most use-cases call for (counters and histograms,
+  for example). However, for Gauges, it's possible to set `MAX` or `MIN` as aggregation, 
+  to get the highest/lowest value of all the processes / threads.
+  
+  Even though this store saves data on disk, it's still much faster than would probably be 
+  expected, because the files are never actually `fsync`ed, so the store never blocks 
+  while waiting for disk. The kernel's page cache is incredibly efficient in this regard.
+  
+  If in doubt, check the benchmark scripts described in the documentation for creating 
+  your own stores and run them in your particular runtime environment to make sure this 
+  provides adequate performance.
+
+### Building your own store, and stores other than the built-in ones.
+
+If none of these stores is suitable for your requirements, you can easily make your own.
+
+The interface and requirements of Stores are specified in detail in the `README.md`
+in the `client/data_stores` directory. This thoroughly documents how to make your own 
+store.
+
+There are also links there to non-built-in stores created by others that may be useful,
+either as they are, or as a starting point for making your own.
+
+### Aggregation settings for multi-process stores
+
+If you are in a multi-process environment (such as pre-fork servers like Unicorn), each
+process will probably keep their own counters, which need to be aggregated when receiving
+a Prometheus scrape, to report coherent total numbers.
+
+For Counters and Histograms (and quantile-less Summaries), this is simply a matter of 
+summing the values of each process.
+
+For Gauges, however, this may not be the right thing to do, depending on what they're 
+measuring. You might want to take the maximum or minimum value observed in any process,
+rather than the sum of all of them.
+
+In those cases, you should use the `store_settings` parameter when registering the 
+metric, to specify an `:aggregation` setting. 
+
+```ruby
+free_disk_space = registry.gauge(:free_disk_space_bytes,
+                                docstring: "Free disk space, in bytes",
+                                store_settings: { aggregation: :max })
+```
+
+NOTE: This will only work if the store you're using supports the `:aggregation` setting.
+Of the built-in stores, only `DirectFileStore` does.
+
+Also note that the `:aggregation` setting works for all metric types, not just for gauges. 
+It would be unusual to use it for anything other than gauges, but if your use-case 
+requires it, the store will respect your aggregation wishes.
+
 ## Tests
 
 Install necessary development gems with `bundle install` and run tests with

data/lib/prometheus/client.rb

@@ -1,6 +1,7 @@
 # encoding: UTF-8
 
 require 'prometheus/client/registry'
+require 'prometheus/client/config'
 
 module Prometheus
   # Client is a ruby implementation for a Prometheus compatible client.
@@ -9,5 +10,9 @@ module Prometheus
     def self.registry
       @registry ||= Registry.new
     end
+
+    def self.config
+      @config ||= Config.new
+    end
   end
 end

data/lib/prometheus/client/config.rb

@@ -0,0 +1,15 @@
+# encoding: UTF-8
+
+require 'prometheus/client/data_stores/synchronized'
+
+module Prometheus
+  module Client
+    class Config
+      attr_accessor :data_store
+
+      def initialize
+        @data_store = Prometheus::Client::DataStores::Synchronized.new
+      end
+    end
+  end
+end

data/lib/prometheus/client/counter.rb

@@ -10,17 +10,11 @@ module Prometheus
         :counter
       end
 
-      def increment(labels = {}, by = 1)
+      def increment(by: 1, labels: {})
         raise ArgumentError, 'increment must be a non-negative number' if by < 0
 
         label_set = label_set_for(labels)
-        synchronize { @values[label_set] += by }
-      end
-
-      private
-
-      def default
-        0.0
+        @store.increment(labels: label_set, by: by)
       end
     end
   end

data/lib/prometheus/client/data_stores/README.md

@@ -0,0 +1,306 @@
+# Custom Data Stores
+
+Stores are basically an abstraction over a Hash, whose keys are in turn a Hash of labels
+plus a metric name. The intention behind having different data stores is solving 
+different requirements for different production scenarios, or performance trade-offs.
+
+The most common of these scenarios are pre-fork servers like Unicorn, which have multiple
+separate processes gathering metrics. If each of these had their own store, the metrics
+reported on each Prometheus scrape would be different, depending on which process handles
+the request. Solving this requires some sort of shared storage between these processes, 
+and there are many ways to solve this problem, each with their own trade-offs.
+
+This abstraction allows us to easily plug in the most adequate store for each scenario.
+
+## Interface
+   
+`Store` exposes a `for_metric` method, which returns a store-specific and metric-specific 
+`MetricStore` object, which represents a "view" onto the actual internal storage for one
+particular metric. Each metric / collector object will have a references to this 
+`MetricStore` and interact with it directly. 
+
+The `MetricStore` class must expose `synchronize`, `set`, `increment`, `get` and `all_values`
+methods,  which are explained in the code sample below. Its initializer should be called 
+only by `Store#for_metric`, not directly.
+
+All values stored are `Float`s.
+
+Internally, a `Store` can store the data however it needs to, based on its requirements. 
+For example, a store that needs to work in a multi-process environment needs to have a 
+shared section of memory, via either Files, an MMap, an external database, or whatever the 
+implementor chooses for their particular use case.
+
+Each `Store` / `MetricStore` will also choose how to divide responsibilities over the 
+storage of values. For some use cases, each `MetricStore` may have their own individual 
+storage, whereas for others, the `Store` may own a central storage, and `MetricStore` 
+objects will access it through the `Store`. This depends on the design choices of each `Store`. 
+
+`Store` and `MetricStore` MUST be thread safe. This applies not only to operations on 
+stored values (`set`, `increment`), but `MetricStore` must also expose a `synchronize`
+method that would allow a Metric to increment multiple values atomically (Histograms need
+this, for example).
+
+Ideally, multiple keys should be modifiable simultaneously, but this is not a
+hard requirement.
+
+This is what the interface looks like, in practice:
+
+```ruby
+module Prometheus
+  module Client
+    module DataStores
+      class CustomStore
+      
+        # Return a MetricStore, which provides a view of the internal data store, 
+        # catering specifically to that metric.
+        #
+        # - `metric_settings` specifies configuration parameters for this metric 
+        #   specifically. These may or may not be necessary, depending on each specific
+        #   store and metric. The most obvious example of this is for gauges in 
+        #   multi-process environments, where the developer needs to choose how those 
+        #   gauges will get aggregated between all the per-process values.
+        # 
+        #   The settings that the store will accept, and what it will do with them, are
+        #   100% Store-specific. Each store should document what settings it will accept
+        #   and how to use them, so the developer using that store can pass the appropriate 
+        #   instantiating the Store itself, and the Metrics they declare.
+        #
+        # - `metric_type` is specified in case a store wants to validate that the settings
+        #   are valid for the metric being set up. It may go unused by most Stores
+        #
+        # Even if your store doesn't need these two parameters, the Store must expose them
+        # to make them swappable.   
+        def for_metric(metric_name, metric_type:, metric_settings: {})
+          # Generally, here a Store would validate that the settings passed in are valid,
+          # and raise if they aren't.
+          validate_metric_settings(metric_type: metric_type, 
+                                   metric_settings: metric_settings)
+          MetricStore.new(store: self, 
+                          metric_name: metric_name, 
+                          metric_type: metric_type, 
+                          metric_settings: metric_settings)
+        end
+
+        
+        # MetricStore manages the data for one specific metric. It's generally a view onto
+        # the central store shared by all metrics, but it could also hold the data itself
+        # if that's better for the specific scenario 
+        class MetricStore
+          # This constructor is internal to this store, so the signature doesn't need to
+          # be this. No one other than the Store should be creating MetricStores 
+          def initialize(store:, metric_name:, metric_type:, metric_settings:)
+          end
+
+          # Metrics may need to modify multiple values at once (Histograms do this, for 
+          # example). MetricStore needs to provide a way to synchronize those, in addition
+          # to all of the value modifications being thread-safe without a need for simple 
+          # Metrics to call `synchronize`
+          def synchronize
+            raise NotImplementedError
+          end
+
+          # Store a value for this metric and a set of labels
+          # Internally, may add extra "labels" to disambiguate values between,
+          # for example, different processes
+          def set(labels:, val:)
+            raise NotImplementedError
+          end
+
+          def increment(labels:, by: 1)
+            raise NotImplementedError
+          end
+  
+          # Return a value for a set of labels
+          # Will return the same value stored by `set`, as opposed to `all_values`, which 
+          # may aggregate multiple values.
+          #
+          # For example, in a multi-process scenario, `set` may add an extra internal
+          # label tagging the value with the process id. `get` will return the value for
+          # "this" process ID. `all_values` will return an aggregated value for all 
+          # process IDs.
+          def get(labels:)
+            raise NotImplementedError
+          end
+  
+          # Returns all the sets of labels seen by the Store, and the aggregated value for 
+          # each.
+          # 
+          # In some cases, this is just a matter of returning the stored value.
+          # 
+          # In other cases, the store may need to aggregate multiple values for the same
+          # set of labels. For example, in a multiple process it may need to `sum` the
+          # values of counters from each process. Or for `gauges`, it may need to take the
+          # `max`. This is generally specified in `metric_settings` when calling 
+          # `Store#for_metric`.
+          def all_values
+            raise NotImplementedError
+          end
+        end
+      end
+    end
+  end
+end
+```
+
+## Conventions
+
+- Your store MAY require or accept extra settings for each metric on the call to `for_metric`.
+- You SHOULD validate these parameters to make sure they are correct, and raise if they aren't. 
+- If your store needs to aggregate multiple values for the same metric (for example, in
+  a multi-process scenario), you MUST accept a setting to define how values are aggregated.
+  - This setting MUST be called `:aggregation`
+  - It MUST support, at least, `:sum`, `:max` and `:min`. 
+  - It MAY support other aggregation modes that may apply to your requirements.
+  - It MUST default to `:sum`
+
+## Testing your Store
+
+In order to make it easier to test your store, the basic functionality is tested using
+`shared_examples`:
+
+`it_behaves_like Prometheus::Client::DataStores`
+
+Follow the simple structure in `synchronized_spec.rb` for a starting point.
+
+Note that if your store stores data somewhere other than in-memory (in files, Redis, 
+databases, etc), you will need to do cleanup between tests in a `before` block.
+
+The tests for `DirectFileStore` have a good example at the top of the file. This file also
+has some examples on testing multi-process stores, checking that aggregation between 
+processes works correctly.
+
+## Benchmarking your custom data store
+
+If you are developing your own data store, you probably want to benchmark it to see how
+it compares to the built-in ones, and to make sure it achieves the performance you want.
+
+The Prometheus Ruby Client includes some benchmarks (in the `spec/benchmarks` directory)
+to help you with this, and also with validating that your store works correctly.
+
+The `README` in that directory contains more information what these benchmarks are for,
+and how to use them.
+
+## Extra Stores and Research
+
+In the process of abstracting stores away, and creating the built-in ones, GoCardless
+has created a good amount of research, benchmarks, and experimental stores, which 
+weren't useful to include in this repo, but may be a useful resource or starting point 
+if you are building your own store.
+
+Check out the [GoCardless Data Stores Experiments](gocardless/prometheus-client-ruby-data-stores-experiments) 
+repository for these.
+
+## Sample, imaginary multi-process Data Store
+
+This is just an example of how one could implement a data store, and a clarification on
+the "aggregation" point 
+
+Important: This is a **toy example**, intended simply to show how this could work / how to
+implement these interfaces.
+
+There are some key pieces of code missing, which are fairly uninteresting, this only shows
+the parts that illustrate the idea of storing multiple different values, and aggregating
+them
+
+```ruby
+module Prometheus
+  module Client
+    module DataStores
+      # Stores all the data in a magic data structure that keeps cross-process data, in a
+      # way that all processes can read it, but each can write only to their own set of
+      # keys.
+      # It doesn't care how that works, this is not an actual solution to anything,
+      # just an example of how the interface would work with something like that.
+      #
+      # Metric Settings have one possible key, `aggregation`, which must be one of
+      # `AGGREGATION_MODES`
+      class SampleMagicMultiprocessStore
+        AGGREGATION_MODES = [MAX = :max, MIN = :min, SUM = :sum]
+        DEFAULT_METRIC_SETTINGS = { aggregation: SUM }
+
+        def initialize
+          @internal_store = MagicHashSharedBetweenProcesses.new # PStore, for example
+        end
+
+        def for_metric(metric_name, metric_type:, metric_settings: {})
+          settings = DEFAULT_METRIC_SETTINGS.merge(metric_settings)
+          validate_metric_settings(metric_settings: settings)
+          MetricStore.new(store: self,
+                          metric_name: metric_name,
+                          metric_type: metric_type,
+                          metric_settings: settings)
+        end
+
+        private
+
+        def validate_metric_settings(metric_settings:)
+          raise unless metric_settings.has_key?(:aggregation)
+          raise unless metric_settings[:aggregation].in?(AGGREGATION_MODES)
+        end
+
+        class MetricStore
+          def initialize(store:, metric_name:, metric_type:, metric_settings:)
+            @store = store
+            @internal_store = store.internal_store
+            @metric_name = metric_name
+            @aggregation_mode = metric_settings[:aggregation]
+          end
+
+          def set(labels:, val:)
+            @internal_store[store_key(labels)] = val.to_f
+          end
+
+          def get(labels:)
+            @internal_store[store_key(labels)]
+          end
+
+          def all_values
+            non_aggregated_values = all_store_values.each_with_object({}) do |(labels, v), acc|
+              if labels["__metric_name"] == @metric_name
+                label_set = labels.reject { |k,_| k.in?("__metric_name", "__pid") }
+                acc[label_set] ||= []
+                acc[label_set] << v
+              end
+            end
+
+            # Aggregate all the different values for each label_set
+            non_aggregated_values.each_with_object({}) do |(label_set, values), acc|
+              acc[label_set] = aggregate(values)
+            end
+          end
+
+          private
+
+          def all_store_values
+            # This assumes there's a something common that all processes can write to, and
+            # it's magically synchronized (which is not true of a PStore, for example, but
+            # would of some sort of external data store like Redis, Memcached, SQLite)
+
+            # This could also have some sort of:
+            #    file_list = Dir.glob(File.join(path, '*.db')).sort
+            # which reads all the PStore files / MMapped files, etc, and returns a hash
+            # with all of them together, which then `values` and `label_sets` can use
+          end
+
+          # This method holds most of the key to how this Store works. Adding `_pid` as
+          # one of the labels, we hold each process's value separately, which we can 
+          # aggregate later 
+          def store_key(labels)
+            labels.merge(
+              {
+                "__metric_name" => @metric_name,
+                "__pid" => Process.pid
+              }
+            )
+          end
+
+          def aggregate(values)
+            # This is a horrible way to do this, just illustrating the point
+            values.send(@aggregation_mode)
+          end
+        end
+      end
+    end
+  end
+end
+```