fly_io 0.1.0

data/contracts/machines/source.json

@@ -0,0 +1,20 @@
+{
+  "surface": "machines_rest",
+  "stability": "stable",
+  "base_url": "https://api.machines.dev",
+  "contract_url": "https://docs.machines.dev/openapi.json",
+  "retrieved_at": "2026-08-26T20:30:19Z",
+  "etag": "\"222e4-Y2JlbkeX2XlDU7gTxh4WI2j04MM\"",
+  "last_modified": "Thu, 20 Aug 2026 16:50:52 GMT",
+  "sha256": "9e6a180c40a38bf20b72a8577a8cd93866f47882627b608af753cc4c22f7758f",
+  "openapi": "3.0.1",
+  "api_version": "1.0",
+  "paths": 68,
+  "operations": 98,
+  "unique_operation_ids": 97,
+  "schemas": 173,
+  "notes": [
+    "Machines_update_metadata is shared by PUT and PATCH on the metadata collection.",
+    "The contract declares no security scheme; official Fly docs define Bearer authentication."
+  ]
+}

data/contracts/public_surface_inventory.json

@@ -0,0 +1,164 @@
+{
+  "retrieved_at": "2026-08-26",
+  "source_revisions": {
+    "flyctl": "1f906765dee8e973f6d1dd8fff65b58212e7b425",
+    "fly_go_v0_9_8": "858ec691152ab3faa0b0a0438726bda8144d8734",
+    "fly_docs": "1e8cac5f0fd7a82e66e3b01ece3a10a8ecb2b7e3"
+  },
+  "surfaces": [
+    {
+      "id": "machines_rest_openapi",
+      "classification": "public_documented_control_plane",
+      "stability": "official_versioned_reference_without_an_explicit_compatibility_policy",
+      "base_urls": {
+        "public": "https://api.machines.dev",
+        "private_wireguard": "http://_api.internal:4280"
+      },
+      "authentication": {
+        "official_guide": "Authorization: Bearer <fly_api_token>",
+        "generated_reference_text": "FlyV1 wording with a Bearer example",
+        "flyctl_behavior": "FlyV1 for macaroon tokens; Bearer for OAuth/user tokens",
+        "implementation_note": "Accept raw tokens and explicit preformatted authorization values; never duplicate a scheme"
+      },
+      "authoritative_contract": {
+        "url": "https://docs.machines.dev/openapi.json",
+        "openapi": "3.0.1",
+        "info_version": "1.0",
+        "sha256": "9e6a180c40a38bf20b72a8577a8cd93866f47882627b608af753cc4c22f7758f",
+        "bytes": 140004,
+        "etag": "\"222e4-Y2JlbkeX2XlDU7gTxh4WI2j04MM\"",
+        "last_modified": "Thu, 20 Aug 2026 16:50:52 GMT"
+      },
+      "counts": {
+        "paths": 68,
+        "operations": 98,
+        "unique_operation_ids": 97,
+        "schemas": 173,
+        "ref_occurrences": 315,
+        "unique_refs": 173,
+        "unresolved_refs": 0,
+        "methods": {"GET": 33, "POST": 43, "PUT": 4, "PATCH": 2, "DELETE": 16},
+        "families": {
+          "Apps": 8,
+          "TLS Certificates": 8,
+          "Machines": 29,
+          "Secrets": 14,
+          "Volumes": 8,
+          "Organizations": 2,
+          "Platform": 2,
+          "Postgres Clusters": 22,
+          "Tokens": 5
+        }
+      },
+      "contract_ambiguities": [
+        "Machines_update_metadata is duplicated for PUT and PATCH, so method plus path must be part of the operation key",
+        "The OpenAPI document defines neither securitySchemes nor top-level security",
+        "The generated landing page omits Organizations and Platform resource cards even though the operations exist",
+        "The top-level tags array omits several tags used by operations",
+        "Official Network Policies operations are absent from this contract"
+      ],
+      "official_sources": [
+        "https://fly.io/docs/machines/api/",
+        "https://docs.machines.dev/",
+        "https://fly.io/docs/machines/api/working-with-machines-api",
+        "https://fly.io/docs/machines/guides-examples/managing-machines-with-the-api"
+      ]
+    },
+    {
+      "id": "machines_network_policies",
+      "classification": "public_documented_control_plane_without_machine_readable_contract",
+      "stability": "no_explicit_stability_label",
+      "base_url": "https://api.machines.dev",
+      "authentication": "Authorization: Bearer <FLY_API_TOKEN>",
+      "authoritative_contract": null,
+      "prose_source": "https://fly.io/docs/machines/guides-examples/network-policies",
+      "operation_count": 3,
+      "operation_inventory": "network_policies_operations.json",
+      "openapi_status": "absent"
+    },
+    {
+      "id": "control_plane_graphql",
+      "classification": "public_endpoint_internal_unstable_control_plane",
+      "stability": "explicitly_no_stability_guarantees_and_may_change_without_notice",
+      "endpoint": "https://api.fly.io/graphql",
+      "authentication": "FlyV1 or Bearer as selected by token type in flyctl; official curl examples show Bearer",
+      "authoritative_contract": null,
+      "implementation_evidence": {
+        "flyctl_schema_path": "gql/schema.graphql",
+        "flyctl_schema_sha256": "2f1a1d0f2f8940277e5e14002c78cb4f6404b3ea1ce9d0999d4d3b65c52f9480",
+        "flyctl_schema_bytes": 190611,
+        "flyctl_schema_lines": 10995,
+        "flyctl_schema_root_query_fields": 39,
+        "flyctl_schema_root_mutation_fields": 141,
+        "schema_last_change_revision": "acb4fa7a4ea6bad1d9defdf36a8db3be62c6d7b7",
+        "schema_provenance": "Manually copied from Fly's internal web repository according to flyctl CONTRIBUTING.md",
+        "flyctl_named_documents": 27,
+        "flyctl_ad_hoc_documents": 2,
+        "fly_go_source_documents": 85,
+        "fly_go_unique_root_fields": 52
+      },
+      "inventories": [
+        "flyctl_graphql_operations.json",
+        "fly_go_graphql_operations.json",
+        "official_docs_graphql_examples.json"
+      ],
+      "official_sources": [
+        "https://fly.io/docs/machines/guides-examples/managing-machines-with-the-api",
+        "https://fly.io/docs/mcp/deploy-with/machines-api",
+        "https://fly.io/docs/blueprints/using-the-fly-docker-registry"
+      ]
+    },
+    {
+      "id": "prometheus_metrics",
+      "classification": "public_documented_observability_data_api_not_control_plane",
+      "stability": "stable_standard_http_query_api_with_qualified_fly_compatibility",
+      "base_url": "https://api.fly.io/prometheus/{org_slug}/",
+      "authentication": "Authorization: Bearer or FlyV1 depending on token type",
+      "authoritative_contract": "Prometheus HTTP API and VictoriaMetrics compatibility documentation; no Fly-owned OpenAPI",
+      "documented_endpoint_families": 7,
+      "endpoint_families": [
+        "/api/v1/query",
+        "/api/v1/query_range",
+        "/api/v1/series",
+        "/api/v1/labels",
+        "/api/v1/label/{label_name}/values",
+        "/api/v1/status/tsdb",
+        "/federate"
+      ],
+      "official_source": "https://fly.io/docs/monitoring/metrics"
+    },
+    {
+      "id": "extensions_partner_oauth",
+      "classification": "partner_gated_public_integration_api",
+      "stability": "no_explicit_stability_label",
+      "base_url": "https://api.fly.io",
+      "authentication": "OAuth client credentials plus partner-provided shared secrets; token info uses Bearer",
+      "authoritative_contract": null,
+      "prose_operations": [
+        "GET /oauth/authorize",
+        "POST /oauth/token",
+        "GET /oauth/token/info",
+        "POST /api/hooks/extensions/{provider}"
+      ],
+      "limitations": "Requires enrollment and Fly-provided credentials; page is excluded from sitemap; not a general account control-plane API",
+      "official_source": "https://fly.io/docs/reference/extensions_api"
+    },
+    {
+      "id": "flyctl_uiex_rest",
+      "classification": "observed_internal_implementation_not_public_contract",
+      "stability": "undocumented_internal",
+      "base_url": "https://api.fly.io/api/v1",
+      "authoritative_contract": null,
+      "flyctl_request_call_sites": 52,
+      "methods": {"GET": 21, "POST": 20, "PATCH": 3, "DELETE": 8},
+      "decision": "Exclude from supported public surface; do not infer stability from flyctl use"
+    },
+    {
+      "id": "tigris_s3_and_fly_registry",
+      "classification": "adjacent_data_planes",
+      "stability": "Tigris Fly documentation is beta; registry uses external Docker/OCI behavior",
+      "authoritative_contract": "External S3 and OCI/Distribution contracts, not Fly control-plane contracts",
+      "decision": "Document as out of the control-plane client boundary"
+    }
+  ]
+}

data/contracts/research_metadata.json

@@ -0,0 +1,17 @@
+{
+  "retrieved_at": "2026-08-26",
+  "rubygems_name": {
+    "name": "fly_io",
+    "api_url": "https://rubygems.org/api/v1/gems/fly_io.json",
+    "api_status": 404,
+    "exact_search_matches": [],
+    "decision": "Use fly_io; availability is verified but only a publish can reserve it."
+  },
+  "ruby": {
+    "authoritative_url": "https://www.ruby-lang.org/en/downloads/releases/",
+    "latest_stable": "4.0.6",
+    "released_at": "2026-07-14",
+    "source_url": "https://cache.ruby-lang.org/pub/ruby/4.0/ruby-4.0.6.tar.gz",
+    "source_sha256": "837d299e8f7ddf2be31a229a7a7e019d354979825117989acb3b32b1a9be262a"
+  }
+}

data/contracts/sources/metrics.html.md

@@ -0,0 +1,474 @@
+---
+title: Metrics on Fly.io
+layout: docs
+nav: firecracker
+redirect_from:
+  - /docs/reference/metrics/
+  - /docs/going-to-production/monitoring/publishing-metrics/
+  - /docs/metrics-and-logs/metrics
+---
+
+<figure>
+  <img src="/static/images/monitor-machines.png" alt="Illustration by Annie Ruygt of some Machines strutting along" class="w-full max-w-lg mx-auto">
+</figure>
+
+The Fly.io platform includes a fully-managed metrics solution to help you easily monitor your Fly apps.
+It includes the following components:
+
+- [**Prometheus on Fly.io**](#prometheus-on-fly-io): Managed Prometheus-compatible time series storage
+- [**Dashboards**](#dashboards): Managed Grafana with detailed visualizations of all built-in metrics
+- [**Built-in Metrics**](#built-in-metrics): Metrics automatically sent from every Fly app you deploy
+- [**Custom Metrics**](#custom-metrics): Expose additional metrics from Fly apps for further customization
+
+## Prometheus on Fly.io
+
+[Prometheus](https://prometheus.io/) is a popular open source monitoring system used to store and query metrics efficiently,
+with a stable [HTTP querying API](https://prometheus.io/docs/prometheus/latest/querying/api/) compatible with a range of systems.
+
+**Prometheus on Fly.io** is a fully-managed service based on [VictoriaMetrics](https://victoriametrics.com/).
+It [supports](https://docs.victoriametrics.com/#prometheus-querying-api-usage) most common Prometheus querying API endpoints:
+- [`/api/v1/query`](https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries)
+- [`/api/v1/query_range`](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries)
+- [`/api/v1/series`](https://prometheus.io/docs/prometheus/latest/querying/api/#finding-series-by-label-matchers)
+- [`/api/v1/labels`](https://prometheus.io/docs/prometheus/latest/querying/api/#getting-label-names)
+- [`/api/v1/label/<label_name>/values`](https://prometheus.io/docs/prometheus/latest/querying/api/#querying-label-values)
+- [`/api/v1/status/tsdb`](https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-stats)
+- [`/federate`](https://prometheus.io/docs/prometheus/latest/federation/)
+
+Note that [remote read](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_read) (`/api/v1/read`) [remote storage integration](https://prometheus.io/docs/prometheus/latest/storage/#remote-storage-integrations)
+is [not supported](https://docs.victoriametrics.com/FAQ.html#why-doesnt-victoriametrics-support-the-prometheus-remote-read-api).
+
+### MetricsQL
+
+Prometheus queries are typically based on the [PromQL](https://prometheus.io/docs/prometheus/latest/querying/basics/) query language.
+Prometheus on Fly.io queries use VictoriaMetrics [MetricsQL](https://docs.victoriametrics.com/MetricsQL.html),
+a backwards-compatible query language that fixes user experience issues and adds
+useful features and functions on top of PromQL.
+
+Key features:
+
+- [Better `rate()` and `increase()`](https://medium.com/@romanhavronenko/victoriametrics-promql-compliance-d4318203f51e#cade)
+functions that just work. No need for [`irate` workarounds](https://www.percona.com/blog/2020/02/28/better-prometheus-rate-function-with-victoriametrics/)
+or appending Grafana's [magical `$__rate_interval`](https://grafana.com/blog/2020/09/28/new-in-grafana-7.2-__rate_interval-for-prometheus-rate-queries-that-just-work/) selector to every query.
+In fact, you can even omit the square brackets entirely and MetricsQL will do the right thing.
+- Many more [label manipulation functions](https://docs.victoriametrics.com/MetricsQL.html#label-manipulation-functions)
+such as `drop_common_labels`, `label_set`, etc.
+- [`topk_avg`](https://docs.victoriametrics.com/MetricsQL.html#topk_avg), which returns the top `k` time series averaged
+across the entire series range (not just individual points), plus the sum of all remaining series in an "other" label.
+Useful for giving a small, filtered view across a potentially large number of series.
+
+### Querying
+
+Queries can be sent to the following endpoint:
+```
+https://api.fly.io/prometheus/<org-slug>/
+```
+
+You’ll need to authenticate with a [Fly Access Token](https://fly.io/docs/security/tokens/). Depending on the token type, the header uses either the Bearer format (`Authorization: Bearer <token>`) or the FlyV1 format (`Authorization: FlyV1 <token>`). You may only query series scoped to your organizations.
+
+
+#### Manually
+
+**Find your Organization slug**
+
+List your organizations, find the org slug and set it as a local variable.
+
+```cmd
+flyctl orgs list
+ORG_SLUG=[org-slug]
+```
+
+**Get an access token**
+
+```cmd
+TOKEN=$(flyctl auth token)
+```
+
+**Test it out!**
+
+```shell
+curl https://api.fly.io/prometheus/$ORG_SLUG/api/v1/query \
+  --data-urlencode 'query=sum(increase(fly_edge_http_responses_count)) by (app, status)' \
+  -H "Authorization: Bearer $TOKEN"
+```
+
+## Retention, cost, and limits
+
+Prometheus on Fly.io retains metric data for approximately 15 days and is intended for operational monitoring. If you need longer retention, run your own monitoring stack. Fly.io's [Observability for User Apps](/docs/blueprints/observability-for-user-apps/) guide deploys a ready-made stack, or you can federate from the Fly.io endpoint into your own Prometheus installation.
+
+There's currently no additional charge for the managed Prometheus and Grafana. Pricing could change in the future, and Fly.io will give advance notice.
+
+A couple of practical limits to be aware of:
+
+- We cap your app's metrics endpoint response at 16 MB and drop anything larger.
+- We may drop very high-cardinality custom metrics.
+
+## Dashboards
+
+For more advanced metrics monitoring, you can use dashboards to organize and visualize complex Prometheus
+queries.
+
+The Metrics tab on the [Fly.io Dashboard](https://fly.io/dashboard) provides an overview
+of your Fly apps using the built-in metrics stored in Prometheus.
+
+### Managed Grafana
+
+[Grafana](https://grafana.com/) is a popular open source data visualization web application, that allows you to
+compose queries against data sources into dynamic, reusable dashboards.
+
+We provide a managed Grafana instance at [fly-metrics.net](https://fly-metrics.net), preconfigured with your
+Prometheus data source and detailed dashboards covering the full set of built-in metrics.
+
+You can also use the Explore panel to run ad-hoc queries against the preconfigured Prometheus datasource,
+or create/import additional dashboards for further customization or to visualize custom metrics.
+
+Switch between your Fly.io Organizations by clicking the "Switch organization" link beneath
+the user icon in the lower-left of the screen.
+
+### External or self-hosted Grafana
+
+You can also configure your Prometheus endpoint with an existing Grafana installation, or [host one on Fly.io](https://github.com/fly-apps/grafana). Either way, you can set it up like this:
+
+1. [Add](https://grafana.com/docs/grafana/latest/datasources/add-a-data-source/) a [Prometheus data source](https://grafana.com/docs/grafana/latest/datasources/prometheus/) (Settings -> Data Sources -> Add data source -> Prometheus)
+2. Fill the form with the following:
+- HTTP -> URL: `https://api.fly.io/prometheus/<org-slug>/`
+- Custom HTTP Headers -> + Add Header:
+   - Header: `Authorization`
+   - Value: `FlyV1 <token>` (for tokens created with `fly tokens create`)
+   - Value: `Bearer <token>` (only for tokens from `flyctl auth token`)
+
+You're all set.
+
+We publish our [Fly.io Dashboards](https://grafana.com/grafana/dashboards/14741) to Grafana.com for use with external Grafana instances.
+To install, just [import the dashboard](https://grafana.com/docs/grafana/latest/dashboards/export-import/#import-dashboard) using the listed IDs.
+If you'd like to contribute changes to the dashboards, we have created a [repository](https://github.com/superfly/dashboards) for them.
+
+## Alerting
+
+Fly.io doesn't include built-in alerting on metrics, so you'll need to set up alerting yourself against the Prometheus endpoint. Two common approaches:
+
+- **Grafana alerting**: connect the Fly.io Prometheus data source to a Grafana instance (see the [External or self-hosted Grafana](#external-or-self-hosted-grafana) section), then create [Grafana alert rules](https://grafana.com/docs/grafana/latest/alerting/) on queries against the data source and send the alerts to a contact point such as email, Slack, or PagerDuty.
+- **Prometheus and Alertmanager**: run your own [Prometheus](https://prometheus.io/) that federates from the Fly.io endpoint and use [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) to send notifications.
+
+If you don't already run Grafana or Prometheus, Fly.io's [Observability for User Apps](/docs/blueprints/observability-for-user-apps/) guide deploys a ready-made stack (VictoriaMetrics and Grafana) you can build alerts on.
+
+## Built-in metrics
+
+Fly apps automatically publish a number of built-in metrics.
+
+[Metric types](https://prometheus.io/docs/concepts/metric_types/) are all [Gauges](https://prometheus.io/docs/concepts/metric_types/#gauge)
+unless otherwise marked.
+
+Metrics with names ending in `_count` are all [Counters](https://prometheus.io/docs/concepts/metric_types/#counter).
+
+[Histogram](https://prometheus.io/docs/concepts/metric_types/#histogram) metrics with a base name of `<name>` expose multiple series:
+- `<name>_bucket{le}`
+- `<name>_sum`
+- `<name>_count`
+
+### Standard Labels
+
+All published series include the following labels:
+
+- `app`: App name
+- `region`: [Fly.io Region](https://fly.io/docs/reference/regions/#fly-io-regions)
+- `host`: 4-character host ID (lowercase hexadecimal)
+- `instance`: App instance ID (for all series except `fly_edge_` and `fly_volume_`)
+
+If your app exposes custom metrics with the same labels, they will be overwritten.
+
+### Proxy series
+
+Any app using a TCP-based handler (HTTP, TLS or straight TCP) publishes `edge` and `app` proxy metrics:
+
+Labels:
+- `proxy_id`: "blue" or "green" (flips when the proxy is restarted/updated)
+
+#### Edge - `fly_edge_`
+
+```
+fly_edge_http_responses_count{status}
+fly_edge_http_response_time_seconds{status} (Histogram)
+fly_edge_tcp_connects_count
+fly_edge_tcp_disconnects_count
+fly_edge_data_out (Counter, bytes)
+fly_edge_data_in (Counter, bytes)
+fly_edge_tls_handshake_errors{servername} (Counter)
+fly_edge_tls_handshake_time_seconds{version} (Histogram)
+```
+
+#### App - `fly_app_`
+```
+fly_app_concurrency
+fly_app_http_responses_count{status}
+fly_app_http_response_time_seconds{status} (Histogram)
+fly_app_connect_time_seconds (Histogram)
+fly_app_tcp_connects_count
+fly_app_tcp_disconnects_count
+```
+
+### Instance series - `fly_instance_`
+
+Derived from the [`/proc` file system](https://www.kernel.org/doc/html/latest/filesystems/proc.html) of your app VMs.
+
+`fly_instance_up = 1` shows the VM is reporting correctly.
+
+### Instance exit - `fly_instance_exit_`
+
+Information about instance exits. These metrics help you understand why your instances are terminating and can be used for alerting and debugging.
+
+- `fly_instance_exit_code`: The exit code of the main process when the instance terminates. A value of 0 typically indicates normal termination, while non-zero values indicate errors or abnormal termination.
+- `fly_instance_exit_oom`: A boolean flag (0 or 1) indicating whether the instance was killed due to out-of-memory (OOM). A value of 1 means the instance was terminated because it exceeded its memory limits.
+- `fly_instance_exit_vm_code`: The VM-level exit code, which may differ from the application exit code. This can help distinguish between application-level failures and VM-level issues.
+
+#### Instance memory - `fly_instance_memory_`
+
+Derived from [`/proc/meminfo`](https://www.kernel.org/doc/html/latest/filesystems/proc.html#meminfo). All units are in *bytes*.
+
+```
+fly_instance_memory_mem_total
+fly_instance_memory_mem_free
+fly_instance_memory_mem_available
+fly_instance_memory_buffers
+fly_instance_memory_cached
+fly_instance_memory_swap_cached
+fly_instance_memory_active
+fly_instance_memory_inactive
+fly_instance_memory_swap_total
+fly_instance_memory_swap_free
+fly_instance_memory_dirty
+fly_instance_memory_writeback
+fly_instance_memory_slab
+fly_instance_memory_shmem
+fly_instance_memory_vmalloc_total
+fly_instance_memory_vmalloc_used
+fly_instance_memory_vmalloc_chunk
+```
+#### Instance Load and CPU
+
+- `load_average` is derived from [`/proc/loadavg`](https://www.kernel.org/doc/html/latest/filesystems/proc.html#id11) ([`getloadavg`](https://man7.org/linux/man-pages/man3/getloadavg.3.html)). It's a ["system load average"](https://www.brendangregg.com/blog/2017-08-08/linux-load-averages.html) measuring the number of processes in the system run queue, with samples representing averages over 1, 5, and 15 `minutes`.
+
+- `cpu` is derived from [`/proc/stat`](https://www.kernel.org/doc/html/latest/filesystems/proc.html#miscellaneous-kernel-statistics-in-proc-stat),
+and counts the amount of time each CPU (`cpu_id`) has spent performing different kinds of work (`mode`, which may be one of `user`, `nice`, `system`, `idle`, `iowait`, `irq`, `softirq`, `steal`, `guest`, `guest_nice`).
+The time unit is 'clock ticks' of centiseconds (0.01 seconds).
+
+The following CPU metrics are related to [CPU Performance](/docs/machines/cpu-performance):
+- `cpu_baseline` is the baseline quota in number of CPUs, calculated from the CPU type and number of vCPUs.
+- `cpu_balance` is the accrued CPU burst balance in clock ticks (centiseconds).
+- `cpu_throttle` is derived from the `throttled_time` field of the cgroup [`cpu.stat`](https://docs.kernel.org/scheduler/sched-bwc.html#statistics), and counts the amount of time the CPU was throttled after exhausting its quota, in 'clock ticks' (centiseconds).
+
+```
+fly_instance_load_average{minutes}
+fly_instance_cpu{cpu_id, mode} (Counter, centiseconds)
+fly_instance_cpu_baseline (CPUs)
+fly_instance_cpu_balance (centiseconds)
+fly_instance_cpu_throttle (Counter, centiseconds)
+```
+#### Instance Disks - `fly_instance_disk_`
+
+Counters derived from fields 1-11 of [`/proc/diskstats`](https://www.kernel.org/doc/html/latest/admin-guide/iostats.html). The unit for `time_` series is **milliseconds**, and the unit for `sectors_` is 512-byte sectors.
+
+Labels:
+- `device`: Published for the ephemeral VM root disk (`vdb`) and any mounted Volume (`vdc`).
+
+```
+fly_instance_disk_reads_completed
+fly_instance_disk_reads_merged
+fly_instance_disk_sectors_read
+fly_instance_disk_time_reading
+fly_instance_disk_writes_completed
+fly_instance_disk_writes_merged
+fly_instance_disk_sectors_written
+fly_instance_disk_time_writing
+fly_instance_disk_io_in_progress
+fly_instance_disk_time_io
+fly_instance_disk_time_io_weighted
+```
+
+#### Instance Networking - `fly_instance_net_`
+
+Counters derived from [`/proc/net/dev`](https://www.kernel.org/doc/html/latest/networking/statistics.html#procfs).
+
+Labels:
+- `device`: interface name, either `eth0` or `dummy0` (ignore).
+
+```
+fly_instance_net_recv_bytes
+fly_instance_net_recv_packets
+fly_instance_net_recv_errs
+fly_instance_net_recv_drop
+fly_instance_net_recv_fifo
+fly_instance_net_recv_frame
+fly_instance_net_recv_compressed
+fly_instance_net_recv_multicast
+fly_instance_net_sent_bytes
+fly_instance_net_sent_packets
+fly_instance_net_sent_errs
+fly_instance_net_sent_drop
+fly_instance_net_sent_fifo
+fly_instance_net_sent_colls
+fly_instance_net_sent_carrier
+fly_instance_net_sent_compressed
+```
+
+#### Instance File Descriptors - `fly_instance_filefd_`
+
+Information about allocated, and maximum allowed allocated file descriptors derived from [`/proc/sys/fs/file-nr`](https://www.kernel.org/doc/html/latest/admin-guide/sysctl/fs.html#file-max-file-nr).
+
+```
+fly_instance_filefd_allocated
+fly_instance_filefd_maximum
+```
+
+#### Instance Filesystem - `fly_instance_filesystem_`
+
+Filesystem metrics derived from [VFS File System Information](https://man7.org/linux/man-pages/man0/sys_statvfs.h.0p.html).
+
+Labels:
+- `mount`: mount point name(s), `/` and if using [Volumes](https://fly.io/docs/volumes/), the destination name in fly.toml.
+
+```
+fly_instance_filesystem_blocks
+fly_instance_filesystem_block_size
+fly_instance_filesystem_blocks_free
+fly_instance_filesystem_blocks_avail
+```
+
+### Volumes - `fly_volume_`
+
+Labels:
+- `id`: Volume ID
+
+If you're using [Volumes](https://fly.io/docs/volumes/) for any of your organization's apps, you'll be able to query these series,
+derived from the `LSize` and `Data%` of the volume's [thin LV](https://man7.org/linux/man-pages/man7/lvmthin.7.html).
+
+```
+fly_volume_size_bytes
+fly_volume_used_pct (0-100)
+```
+
+### Postgres - `pg_`
+
+If you have a [Postgres](https://fly.io/docs/reference/postgres/) database hosted on Fly.io, you'll automatically get the following series,
+published via [`postgres_exporter`](https://github.com/prometheus-community/postgres_exporter):
+
+```
+pg_stat_activity_count
+pg_stat_activity_max_tx_duration
+pg_stat_archiver_archived_count
+pg_stat_archiver_failed_count
+pg_stat_bgwriter_buffers_alloc
+pg_stat_bgwriter_buffers_backend_fsync
+pg_stat_bgwriter_buffers_backend
+pg_stat_bgwriter_buffers_checkpoint
+pg_stat_bgwriter_buffers_clean
+pg_stat_bgwriter_checkpoint_sync_time
+pg_stat_bgwriter_checkpoint_write_time
+pg_stat_bgwriter_checkpoints_req
+pg_stat_bgwriter_checkpoints_timed
+pg_stat_bgwriter_maxwritten_clean
+pg_stat_bgwriter_stats_reset
+pg_stat_database_blk_read_time
+pg_stat_database_blk_write_time
+pg_stat_database_blks_hit
+pg_stat_database_blks_read
+pg_stat_database_conflicts_confl_bufferpin
+pg_stat_database_conflicts_confl_deadlock
+pg_stat_database_conflicts_confl_lock
+pg_stat_database_conflicts_confl_snapshot
+pg_stat_database_conflicts_confl_tablespace
+pg_stat_database_conflicts
+pg_stat_database_deadlocks
+pg_stat_database_numbackends
+pg_stat_database_stats_reset
+pg_stat_database_tup_deleted
+pg_stat_database_tup_fetched
+pg_stat_database_tup_inserted
+pg_stat_database_tup_returned
+pg_stat_database_tup_updated
+pg_stat_database_xact_commit
+pg_stat_database_xact_rollback
+pg_stat_replication_pg_current_wal_lsn_bytes
+pg_stat_replication_pg_wal_lsn_diff
+pg_stat_replication_reply_time
+pg_replication_lag
+pg_database_size_bytes
+```
+
+## Custom Metrics
+
+For further customization beyond built-in metrics,
+Fly apps can expose a metrics endpoint we'll automatically scrape every 15 seconds and
+send the results to Prometheus.
+
+### Configuration
+
+Add a `[metrics]` section to your application's `fly.toml`:
+
+```toml
+[metrics]
+port = 9091
+path = "/metrics" # default for most prometheus exporters
+```
+
+If your app uses [multiple processes](/docs/apps/processes/), you can add multiple `[[metrics]]` sections, each with its own set of `processes`:
+
+```toml
+[[metrics]]
+port = 9394
+path = "/metrics"
+processes = ["web"]
+
+[[metrics]]
+port = 9113
+path = "/metrics"
+processes = ["proxy"]
+```
+
+### Instrumentation
+
+Instrument your app and expose your metrics on `0.0.0.0`.
+
+There are many supported [client libraries](https://prometheus.io/docs/instrumenting/clientlibs/) as well as off-the-shelf [exporters](https://prometheus.io/docs/instrumenting/exporters/) able to return Prometheus-formatted metrics.
+
+## Authentication
+
+Authenticating to the Prometheus API can be achieved a few different ways, depending on the level of access you want your token to have.
+
+### Fly Access Token
+
+As in the earlier example, a full access token can be generated with `flyctl auth token` and then passed as a bearer token in the `Authorization` header. The header looks like:
+
+```
+Authorization: Bearer THE_TOKEN
+```
+
+### Fly org-restricted or read-only token
+
+This kind of token or "[macaroon](https://fly.io/blog/macaroons-escalated-quickly/)" can be scoped to a single organization or configured to only allow read operations, which can be safer than using a full-blown read-write token that grants access to all organizations under your account.
+
+### Generating tokens
+Create an org-restricted token:
+
+```
+fly tokens create org -o THE_ORGANIZATION
+```
+
+Create a read-only org-restricted token:
+
+```
+fly tokens create readonly
+```
+
+These tokens look like this once generated: `FlyV1 fm2_lJPECAAAAAAAAC7txBAzYI6PRWhHLT...(a lot of base64-encoded text)`.
+
+#### Using tokens
+Use the correct scheme based on token type:
+
+- `Bearer <token>` → for `flyctl auth token`.
+- `FlyV1 <token>` → for all tokens created with `fly tokens create`.
+
+Example with a token created via `fly tokens create`:
+```
+Authorization: FlyV1 fm2_lJPECAAAAAAAAC7txBAzYI6PRWhHLT...(a lot of base64-encoded text)
+```