ferrogw 1.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +531 -0
- package/bin/ferrogw.js +105 -0
- package/package.json +44 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
6
|
+
|
|
7
|
+
1. Definitions.
|
|
8
|
+
|
|
9
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
10
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
11
|
+
|
|
12
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
13
|
+
the copyright owner that is granting the License.
|
|
14
|
+
|
|
15
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
16
|
+
other entities that control, are controlled by, or are under common
|
|
17
|
+
control with that entity. For the purposes of this definition,
|
|
18
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
19
|
+
direction or management of such entity, whether by contract or
|
|
20
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
21
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
22
|
+
|
|
23
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
24
|
+
exercising permissions granted by this License.
|
|
25
|
+
|
|
26
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
27
|
+
including but not limited to software source code, documentation
|
|
28
|
+
source, and configuration files.
|
|
29
|
+
|
|
30
|
+
"Object" form shall mean any form resulting from mechanical
|
|
31
|
+
transformation or translation of a Source form, including but
|
|
32
|
+
not limited to compiled object code, generated documentation,
|
|
33
|
+
and conversions to other media types.
|
|
34
|
+
|
|
35
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
36
|
+
Object form, made available under the License, as indicated by a
|
|
37
|
+
copyright notice that is included in or attached to the work
|
|
38
|
+
(an example is provided in the Appendix below).
|
|
39
|
+
|
|
40
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
41
|
+
form, that is based on (or derived from) the Work and for which the
|
|
42
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
43
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
44
|
+
of this License, Derivative Works shall not include works that remain
|
|
45
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
46
|
+
the Work and Derivative Works thereof.
|
|
47
|
+
|
|
48
|
+
"Contribution" shall mean any work of authorship, including
|
|
49
|
+
the original version of the Work and any modifications or additions
|
|
50
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
51
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
52
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
53
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
54
|
+
means any form of electronic, verbal, or written communication sent
|
|
55
|
+
to the Licensor or its representatives, including but not limited to
|
|
56
|
+
communication on electronic mailing lists, source code control systems,
|
|
57
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
58
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
59
|
+
excluding communication that is conspicuously marked or otherwise
|
|
60
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
61
|
+
|
|
62
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
63
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
64
|
+
subsequently incorporated within the Work.
|
|
65
|
+
|
|
66
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
67
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
68
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
69
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
70
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
71
|
+
Work and such Derivative Works in Source or Object form.
|
|
72
|
+
|
|
73
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
74
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
75
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
76
|
+
(except as stated in this section) patent license to make, have made,
|
|
77
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
78
|
+
where such license applies only to those patent claims licensable
|
|
79
|
+
by such Contributor that are necessarily infringed by their
|
|
80
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
81
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
82
|
+
institute patent litigation against any entity (including a
|
|
83
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
84
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
85
|
+
or contributory patent infringement, then any patent licenses
|
|
86
|
+
granted to You under this License for that Work shall terminate
|
|
87
|
+
as of the date such litigation is filed.
|
|
88
|
+
|
|
89
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
90
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
91
|
+
modifications, and in Source or Object form, provided that You
|
|
92
|
+
meet the following conditions:
|
|
93
|
+
|
|
94
|
+
(a) You must give any other recipients of the Work or
|
|
95
|
+
Derivative Works a copy of this License; and
|
|
96
|
+
|
|
97
|
+
(b) You must cause any modified files to carry prominent notices
|
|
98
|
+
stating that You changed the files; and
|
|
99
|
+
|
|
100
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
101
|
+
that You distribute, all copyright, patent, trademark, and
|
|
102
|
+
attribution notices from the Source form of the Work,
|
|
103
|
+
excluding those notices that do not pertain to any part of
|
|
104
|
+
the Derivative Works; and
|
|
105
|
+
|
|
106
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
107
|
+
distribution, then any Derivative Works that You distribute must
|
|
108
|
+
include a readable copy of the attribution notices contained
|
|
109
|
+
within such NOTICE file, excluding those notices that do not
|
|
110
|
+
pertain to any part of the Derivative Works, in at least one
|
|
111
|
+
of the following places: within a NOTICE text file distributed
|
|
112
|
+
as part of the Derivative Works; within the Source form or
|
|
113
|
+
documentation, if provided along with the Derivative Works; or,
|
|
114
|
+
within a display generated by the Derivative Works, if and
|
|
115
|
+
wherever such third-party notices normally appear. The contents
|
|
116
|
+
of the NOTICE file are for informational purposes only and
|
|
117
|
+
do not modify the License. You may add Your own attribution
|
|
118
|
+
notices within Derivative Works that You distribute, alongside
|
|
119
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
120
|
+
that such additional attribution notices cannot be construed
|
|
121
|
+
as modifying the License.
|
|
122
|
+
|
|
123
|
+
You may add Your own copyright statement to Your modifications and
|
|
124
|
+
may provide additional or different license terms and conditions
|
|
125
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
126
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
127
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
128
|
+
the conditions stated in this License.
|
|
129
|
+
|
|
130
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
131
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
132
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
133
|
+
this License, without any additional terms or conditions.
|
|
134
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
135
|
+
the terms of any separate license agreement you may have executed
|
|
136
|
+
with Licensor regarding such Contributions.
|
|
137
|
+
|
|
138
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
139
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
140
|
+
except as required for reasonable and customary use in describing the
|
|
141
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
142
|
+
|
|
143
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
144
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
145
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
146
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
147
|
+
implied, including, without limitation, any warranties or conditions
|
|
148
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
149
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
150
|
+
appropriateness of using or redistributing the Work and assume any
|
|
151
|
+
risks associated with Your exercise of permissions under this License.
|
|
152
|
+
|
|
153
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
154
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
155
|
+
unless required by applicable law (such as deliberate and grossly
|
|
156
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
157
|
+
liable to You for damages, including any direct, indirect, special,
|
|
158
|
+
incidental, or consequential damages of any character arising as a
|
|
159
|
+
result of this License or out of the use or inability to use the
|
|
160
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
161
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
162
|
+
other commercial damages or losses), even if such Contributor
|
|
163
|
+
has been advised of the possibility of such damages.
|
|
164
|
+
|
|
165
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
166
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
167
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
168
|
+
or other liability obligations and/or rights consistent with this
|
|
169
|
+
License. However, in accepting such obligations, You may act only
|
|
170
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
171
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
172
|
+
defend, and hold each Contributor harmless for any liability
|
|
173
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
174
|
+
of your accepting any such warranty or additional liability.
|
|
175
|
+
|
|
176
|
+
END OF TERMS AND CONDITIONS
|
|
177
|
+
|
|
178
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
179
|
+
|
|
180
|
+
To apply the Apache License to your work, attach the following
|
|
181
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
182
|
+
replaced with your own identifying information. (Don't include
|
|
183
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
184
|
+
comment syntax for the file format. We also recommend that a
|
|
185
|
+
file or class name and description of purpose be included on the
|
|
186
|
+
same "printed page" as the copyright notice for easier
|
|
187
|
+
identification within third-party archives.
|
|
188
|
+
|
|
189
|
+
Copyright [yyyy] [name of copyright owner]
|
|
190
|
+
|
|
191
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
|
+
you may not use this file except in compliance with the License.
|
|
193
|
+
You may obtain a copy of the License at
|
|
194
|
+
|
|
195
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
196
|
+
|
|
197
|
+
Unless required by applicable law or agreed to in writing, software
|
|
198
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
199
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
200
|
+
See the License for the specific language governing permissions and
|
|
201
|
+
limitations under the License.
|
package/README.md
ADDED
|
@@ -0,0 +1,531 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
|
|
3
|
+
<p align="right">
|
|
4
|
+
<a href="README.md">English</a> | <a href="README.zh-CN.md">中文</a>
|
|
5
|
+
</p>
|
|
6
|
+
|
|
7
|
+
<table border="0" cellspacing="0" cellpadding="0"><tr>
|
|
8
|
+
<td rowspan="2"><img src="docs/logo.png" alt="Ferro Labs AI Gateway" width="64" /></td>
|
|
9
|
+
<td align="center"><h1>Ferro Labs AI Gateway</h1></td>
|
|
10
|
+
</tr><tr>
|
|
11
|
+
<td align="center"><strong>Open-Source, OpenAI-Compatible LLM Gateway</strong></td>
|
|
12
|
+
</tr></table>
|
|
13
|
+
|
|
14
|
+
**High-performance AI gateway in Go. Route LLM requests across 30 providers via a single OpenAI-compatible API.**
|
|
15
|
+
|
|
16
|
+
[](https://railway.com/deploy/ferro-labs-ai-sqlite-storage?referralCode=KblxKX&utm_medium=integration&utm_source=template&utm_campaign=generic)
|
|
17
|
+
[](https://railway.com/deploy/ferro-labs-ai-postgresql-storage?referralCode=KblxKX&utm_medium=integration&utm_source=template&utm_campaign=generic)
|
|
18
|
+
[](https://render.com/deploy?repo=https://github.com/ferro-labs/ai-gateway)
|
|
19
|
+
|
|
20
|
+
[](https://go.dev)
|
|
21
|
+
[](https://pkg.go.dev/github.com/ferro-labs/ai-gateway)
|
|
22
|
+
[](https://codecov.io/gh/ferro-labs/ai-gateway)
|
|
23
|
+
[](LICENSE)
|
|
24
|
+
[](https://github.com/ferro-labs/ai-gateway/stargazers)
|
|
25
|
+
[](https://github.com/ferro-labs/ai-gateway/actions/workflows/ci.yml)
|
|
26
|
+
[](https://github.com/ferro-labs/ai-gateway/actions/workflows/code-scanning.yml)
|
|
27
|
+
[](https://deepwiki.com/ferro-labs/ai-gateway)
|
|
28
|
+
[](https://artifacthub.io/packages/search?org=ferro-labs)
|
|
29
|
+
[](https://docs.ferrolabs.ai)
|
|
30
|
+
[](https://discord.gg/yCAeYvJeDV)
|
|
31
|
+
|
|
32
|
+
📖 **Documentation:** [docs.ferrolabs.ai](https://docs.ferrolabs.ai)
|
|
33
|
+
|
|
34
|
+
🔀 **30 providers, 2,500+ models — one API**<br/>
|
|
35
|
+
⚡ **13,925 RPS at 1,000 concurrent users** ([v1.0.0 benchmark](#performance))<br/>
|
|
36
|
+
📦 **Single static binary, no external services required, 32 MB base memory**
|
|
37
|
+
|
|
38
|
+
<img src="docs/architecture.svg" alt="Ferro Labs AI Gateway Architecture" width="100%" />
|
|
39
|
+
|
|
40
|
+
</div>
|
|
41
|
+
|
|
42
|
+
---
|
|
43
|
+
|
|
44
|
+
## Quick Start
|
|
45
|
+
|
|
46
|
+
Get from zero to first request in under 2 minutes.
|
|
47
|
+
|
|
48
|
+
### Option A — Binary (fastest)
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
VER=$(curl -fsSL https://api.github.com/repos/ferro-labs/ai-gateway/releases/latest | grep '"tag_name"' | cut -d'"' -f4)
|
|
52
|
+
curl -fsSL "https://github.com/ferro-labs/ai-gateway/releases/download/${VER}/ferrogw_${VER#v}_linux_amd64.tar.gz" | tar xz
|
|
53
|
+
chmod +x ferrogw
|
|
54
|
+
./ferrogw init # generates config.yaml + MASTER_KEY
|
|
55
|
+
export GATEWAY_CONFIG=./config.yaml # the server reads a config file only when this is set
|
|
56
|
+
export OPENAI_API_KEY=sk-your-key # providers are registered at startup, so export before starting
|
|
57
|
+
export MASTER_KEY=fgw_your-master-key # the key ferrogw init printed
|
|
58
|
+
./ferrogw # starts the server
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Releases are signed and ship SBOMs — verification steps are in
|
|
62
|
+
[SECURITY.md](SECURITY.md#verifying-releases).
|
|
63
|
+
|
|
64
|
+
### Option B — Docker
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
docker pull ghcr.io/ferro-labs/ai-gateway:latest
|
|
68
|
+
docker run -p 8080:8080 \
|
|
69
|
+
-e OPENAI_API_KEY=sk-your-key \
|
|
70
|
+
-e MASTER_KEY=fgw_your-master-key \
|
|
71
|
+
ghcr.io/ferro-labs/ai-gateway:latest
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
### Option C — Go
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
go install github.com/ferro-labs/ai-gateway/cmd/ferrogw@latest
|
|
78
|
+
ferrogw init # first-run setup
|
|
79
|
+
export GATEWAY_CONFIG=./config.yaml # the server reads a config file only when this is set
|
|
80
|
+
export OPENAI_API_KEY=sk-your-key # providers are registered at startup, so export before starting
|
|
81
|
+
export MASTER_KEY=fgw_your-master-key # the key ferrogw init printed
|
|
82
|
+
ferrogw # start the server
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
`ferrogw init` generates the master key and writes a minimal `config.yaml`. The
|
|
86
|
+
key is shown **once** and never written to disk — store it in your `.env` file
|
|
87
|
+
or secret manager.
|
|
88
|
+
|
|
89
|
+
<div align="center">
|
|
90
|
+
<img src="docs/demo.gif" alt="Ferro Labs AI Gateway — Quick Start Demo" width="720" />
|
|
91
|
+
</div>
|
|
92
|
+
|
|
93
|
+
### First request
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
export MASTER_KEY=fgw_your-master-key # the key ferrogw init printed, in whichever shell you curl from
|
|
97
|
+
|
|
98
|
+
curl http://localhost:8080/v1/chat/completions \
|
|
99
|
+
-H "Content-Type: application/json" \
|
|
100
|
+
-H "Authorization: Bearer $MASTER_KEY" \
|
|
101
|
+
-d '{
|
|
102
|
+
"model": "gpt-4o-mini",
|
|
103
|
+
"messages": [{"role": "user", "content": "Hello from Ferro Labs AI Gateway"}]
|
|
104
|
+
}' | jq
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
---
|
|
108
|
+
|
|
109
|
+
## Why Ferro Labs AI Gateway
|
|
110
|
+
|
|
111
|
+
Most AI gateways are Python proxies that crack under load or JavaScript services that eat memory. Ferro Labs AI Gateway is written in Go from the ground up for real-world throughput — a single binary that routes LLM requests with predictable latency and minimal resource usage.
|
|
112
|
+
|
|
113
|
+
| Feature | Ferro Labs | LiteLLM | Bifrost | Kong AI |
|
|
114
|
+
|:-----------------|:------------|:--------|:-----------|:------------|
|
|
115
|
+
| Language | Go | Python | Go | Go/Lua |
|
|
116
|
+
| Single binary | ✅ | ❌ | ✅ | ❌ |
|
|
117
|
+
| Providers | 30 | 100+ | 20+ | 10+ |
|
|
118
|
+
| MCP support | ✅ | ❌ | ✅ | ❌ |
|
|
119
|
+
| Response cache | ✅ | ✅ | ✅ | ❌ (paid) |
|
|
120
|
+
| Guardrails | ✅ | ✅ | ❌ | ❌ (paid) |
|
|
121
|
+
| OSS license | Apache 2.0 | MIT | Apache 2.0 | Apache 2.0 |
|
|
122
|
+
| Managed cloud | Coming Soon | ✅ | ✅ | ✅ |
|
|
123
|
+
|
|
124
|
+
---
|
|
125
|
+
|
|
126
|
+
## Features
|
|
127
|
+
|
|
128
|
+
| Capability | What it does | Reference |
|
|
129
|
+
|:---|:---|:---|
|
|
130
|
+
| 🔀 **Routing** | 8 strategies — single, fallback, load balance, least latency, cost-optimized, content-based, A/B test, conditional — with per-target retry, failover, and model aliases | [Docs →](https://docs.ferrolabs.ai/routing/) |
|
|
131
|
+
| 🔌 **30 providers** | Chat and streaming everywhere; embeddings, images, rerank, moderations, speech-to-text, text-to-speech, and batch where the vendor offers them | [Docs →](https://docs.ferrolabs.ai/providers/) |
|
|
132
|
+
| 🛡️ **Guardrails & plugins** | Six built in — word filter, token/message limits, response cache, rate limiting, per-key budgets, request logging — and the plugin framework is public for writing your own | [Docs →](https://docs.ferrolabs.ai/plugins/) |
|
|
133
|
+
| 🎯 **Capability matrix** | One declarative record of which OpenAI parameters each provider forwards, translates, or cannot express, served by `GET /v1/capabilities` | [Docs →](https://docs.ferrolabs.ai/guides/provider-capabilities/) |
|
|
134
|
+
| 🤖 **MCP** | Connects to stdio and Streamable HTTP tool servers, injects their tools into chat completions, and drives the agentic `tool_calls` loop itself | [Docs →](https://docs.ferrolabs.ai/guides/mcp/) |
|
|
135
|
+
| 📊 **Observability** | OpenTelemetry tracing and Prometheus metrics, one trace ID across logs and spans — a zero-allocation no-op until enabled | [Docs →](https://docs.ferrolabs.ai/guides/observability/) |
|
|
136
|
+
| 🖥️ **Dashboard** | Operations console compiled into the binary and served at `/` — traffic, spend, provider health, request logs, audit trail | [Docs →](https://docs.ferrolabs.ai/guides/dashboard/) |
|
|
137
|
+
|
|
138
|
+
---
|
|
139
|
+
|
|
140
|
+
## Dashboard
|
|
141
|
+
|
|
142
|
+
Every gateway binary serves a built-in operations console at `/` — same port as
|
|
143
|
+
the API, compiled in with `go:embed`, no second image and no second origin. Sign
|
|
144
|
+
in with your `MASTER_KEY` or any admin / read-only key and it reads the live
|
|
145
|
+
gateway: traffic, spend, provider health, routing, plugins, request logs, and the
|
|
146
|
+
audit trail.
|
|
147
|
+
|
|
148
|
+
<div align="center">
|
|
149
|
+
<img src="docs/dashboard.gif" alt="Ferro Labs AI Gateway operations console: Overview, Analytics, Providers, Routing Strategies, Plugins, Playground, Tracing, Request Logs, Audit Trail, Configuration, and API Keys" width="100%" />
|
|
150
|
+
</div>
|
|
151
|
+
|
|
152
|
+
Overview, Analytics (latency/TTFT/cost percentiles), Providers, Routing,
|
|
153
|
+
Plugins, a Playground over the real routing path, Tracing, Request Logs, the
|
|
154
|
+
Audit Trail, Configuration with history/rollback, and scoped API key management.
|
|
155
|
+
|
|
156
|
+
Run the gateway and open <http://localhost:8080>. To see it filled like the
|
|
157
|
+
recording above, bring up the self-contained demo stack:
|
|
158
|
+
|
|
159
|
+
```bash
|
|
160
|
+
make up-fullstack # gateway + Postgres + Jaeger + Prometheus + Grafana + mock upstream + load generator
|
|
161
|
+
# then open http://localhost:8080
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
Build and development details are in [web/README.md](web/README.md).
|
|
165
|
+
|
|
166
|
+
---
|
|
167
|
+
|
|
168
|
+
## Documentation
|
|
169
|
+
|
|
170
|
+
The root README is the overview; each subsystem keeps its own reference beside
|
|
171
|
+
its code:
|
|
172
|
+
|
|
173
|
+
| Reference | Covers |
|
|
174
|
+
|:---|:---|
|
|
175
|
+
| [providers/README.md](providers/README.md) | The 30 providers, the per-provider endpoint matrix, and every `/v1/*` surface |
|
|
176
|
+
| [config/README.md](config/README.md) | Config loading, validation, `${VAR}` secrets, declared models, trusted proxies |
|
|
177
|
+
| [internal/strategies/README.md](internal/strategies/README.md) | All 8 routing strategies and their failure semantics |
|
|
178
|
+
| [plugin/README.md](plugin/README.md) | The plugin framework and the six built-in plugins |
|
|
179
|
+
| [mcp/README.md](mcp/README.md) | MCP tool servers, transports, the subprocess trust boundary, readiness |
|
|
180
|
+
| [observability/README.md](observability/README.md) | Tracing setup, managed backends, emitted attributes, privacy levels, exporters |
|
|
181
|
+
| [deploy/README.md](deploy/README.md) | Dockerfiles, Compose files, the fullstack demo stack |
|
|
182
|
+
| [web/README.md](web/README.md) | Dashboard development and the embed contract |
|
|
183
|
+
| [AGENTS.md](AGENTS.md) | The complete operator/developer reference: architecture, every env var, request flow |
|
|
184
|
+
| [SECURITY.md](SECURITY.md) | Reporting, security posture, release verification |
|
|
185
|
+
| [CONTRIBUTING.md](CONTRIBUTING.md) | Branch strategy, commit conventions, provider/plugin checklists |
|
|
186
|
+
|
|
187
|
+
---
|
|
188
|
+
|
|
189
|
+
## Examples
|
|
190
|
+
|
|
191
|
+
Integration examples for common use cases are in [ferro-labs/ai-gateway-examples](https://github.com/ferro-labs/ai-gateway-examples):
|
|
192
|
+
|
|
193
|
+
| Example | Description |
|
|
194
|
+
|:--------|:------------|
|
|
195
|
+
| [basic](https://github.com/ferro-labs/ai-gateway-examples/tree/main/basic) | Single chat completion to the first configured provider |
|
|
196
|
+
| [fallback](https://github.com/ferro-labs/ai-gateway-examples/tree/main/fallback) | Fallback strategy — try providers in order with retries |
|
|
197
|
+
| [loadbalance](https://github.com/ferro-labs/ai-gateway-examples/tree/main/loadbalance) | Weighted load balancing across targets (70/30 split) |
|
|
198
|
+
| [with-guardrails](https://github.com/ferro-labs/ai-gateway-examples/tree/main/with-guardrails) | Built-in word-filter and max-token guardrail plugins |
|
|
199
|
+
| [with-mcp](https://github.com/ferro-labs/ai-gateway-examples/tree/main/with-mcp) | Local MCP server with tool-calling integration |
|
|
200
|
+
| [embedded](https://github.com/ferro-labs/ai-gateway-examples/tree/main/embedded) | Embed the gateway as an HTTP handler inside an existing server |
|
|
201
|
+
|
|
202
|
+
---
|
|
203
|
+
|
|
204
|
+
## Configuration
|
|
205
|
+
|
|
206
|
+
One YAML/JSON file, named by `GATEWAY_CONFIG`, drives routing, guardrails, MCP
|
|
207
|
+
tools, and observability:
|
|
208
|
+
|
|
209
|
+
```yaml
|
|
210
|
+
strategy:
|
|
211
|
+
mode: fallback # 8 modes — see internal/strategies/README.md
|
|
212
|
+
|
|
213
|
+
targets:
|
|
214
|
+
- virtual_key: openai # an allowlist: only listed providers are routable
|
|
215
|
+
retry: { attempts: 3 } # per target, honoured under every routing mode
|
|
216
|
+
concurrency: { max_concurrency: 32, queue_size: 1000 }
|
|
217
|
+
- virtual_key: anthropic
|
|
218
|
+
|
|
219
|
+
aliases:
|
|
220
|
+
fast: gpt-4o-mini
|
|
221
|
+
|
|
222
|
+
plugins: # guardrails first, then cache — see plugin/README.md
|
|
223
|
+
- name: word-filter
|
|
224
|
+
type: guardrail
|
|
225
|
+
stage: before_request
|
|
226
|
+
enabled: true
|
|
227
|
+
config: { blocked_words: ["password", "secret"] }
|
|
228
|
+
|
|
229
|
+
mcp_servers: # tool servers — see mcp/README.md
|
|
230
|
+
- name: search
|
|
231
|
+
url: https://mcp.example.com/mcp
|
|
232
|
+
headers: { Authorization: "Bearer ${SEARCH_TOKEN}" }
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
`${VAR}` references (braced form only) resolve when a component is constructed,
|
|
236
|
+
never at file load — so secrets are never stored, served by `GET /admin/config`,
|
|
237
|
+
or restored by a rollback. A bare `$` is data; an undefined variable is an
|
|
238
|
+
error.
|
|
239
|
+
|
|
240
|
+
The full annotated reference with every option is
|
|
241
|
+
[config.example.yaml](config.example.yaml) /
|
|
242
|
+
[config.example.json](config.example.json), and the schema guide is
|
|
243
|
+
[config/README.md](config/README.md). `ferrogw validate` checks a file without
|
|
244
|
+
starting the server.
|
|
245
|
+
|
|
246
|
+
### Key environment variables
|
|
247
|
+
|
|
248
|
+
| Variable | Purpose |
|
|
249
|
+
|----------|---------|
|
|
250
|
+
| `MASTER_KEY` | Bootstrap and break-glass admin credential (generated by `ferrogw init`); give each operator their own key from `POST /admin/keys` for day-to-day use |
|
|
251
|
+
| `GATEWAY_CONFIG` | Path to config YAML/JSON |
|
|
252
|
+
| `GATEWAY_ENV` | Set to `production` to enable production-mode safety guards: it refuses to start on `ALLOW_UNAUTHENTICATED_PROXY=true` or a `*` entry in `CORS_ORIGINS`, and warns when per-IP rate limiting is off, pprof is mounted, or the API key store is in-memory |
|
|
253
|
+
| `PORT` | Server port (default: `8080`) |
|
|
254
|
+
| `ALLOW_UNAUTHENTICATED_PROXY` | Set to `true` to disable proxy-route auth (dev only; blocked when `GATEWAY_ENV=production`) |
|
|
255
|
+
| `CORS_ORIGINS` | Comma-separated allowed CORS origins; cross-origin is denied when unset. Matched **literally** — there is no wildcard, so list each origin explicitly |
|
|
256
|
+
| `TRUSTED_PROXIES` | CIDRs of trusted reverse proxies; forwarded headers are honored only from these (default: loopback). See [config/README.md](config/README.md#trusted-proxies-trusted_proxies) |
|
|
257
|
+
| `<PROVIDER>_BASE_URL` | Points a provider at a proxy, self-hosted server, or regional endpoint. It is the **API root**, used verbatim — write it exactly as the vendor documents it, version segment included (`https://api.groq.com/openai/v1`); a bare host resolves to the provider's own version segment |
|
|
258
|
+
|
|
259
|
+
See [AGENTS.md](AGENTS.md) for the full environment variable reference including provider API keys, store backends, and OTel settings.
|
|
260
|
+
|
|
261
|
+
---
|
|
262
|
+
|
|
263
|
+
## Observability
|
|
264
|
+
|
|
265
|
+
**See everything your gateway does** — every request, what it cost, how long it took, which provider served it, and which guardrails ran. Ferro Labs AI Gateway ships first-class **OpenTelemetry tracing** and **Prometheus metrics** out of the box, and stays at a **zero-allocation no-op until you turn it on**. Point it at **Jaeger, Grafana, New Relic, LangSmith, Datadog, or Honeycomb** — anything that speaks OTLP — and every request emits a `gateway.request` span carrying GenAI semantic conventions (`gen_ai.*`) plus `ferro.*` extensions for cost, routing, MCP tool calls, and stream timings. The same trace ID threads your logs, spans, and the `X-Request-ID` response header.
|
|
266
|
+
|
|
267
|
+
> 📈 **Full observability guide → [observability/README.md](observability/README.md)** — managed-backend setup, endpoint & transport rules, every emitted attribute, privacy levels, and exporter plugins.
|
|
268
|
+
|
|
269
|
+
Bring up the gateway wired to a full monitoring stack — **Prometheus, Grafana, and Jaeger** — driven by generated traffic, in one command:
|
|
270
|
+
|
|
271
|
+
```bash
|
|
272
|
+
make up-fullstack # then open Grafana at http://localhost:3000
|
|
273
|
+
```
|
|
274
|
+
|
|
275
|
+
<p align="center">
|
|
276
|
+
<img src="docs/observability/grafana-dashboard.gif" alt="Grafana dashboard: per-provider request rate, latency percentiles, token cost, and circuit-breaker state" width="100%" />
|
|
277
|
+
<br/>
|
|
278
|
+
<em>Grafana — request rate, latency percentiles, per-provider breakdown, token cost, and circuit-breaker state, all from the gateway's Prometheus metrics.</em>
|
|
279
|
+
</p>
|
|
280
|
+
|
|
281
|
+
<p align="center">
|
|
282
|
+
<img src="docs/observability/jaeger-trace.gif" alt="Jaeger trace: one gateway.request span expanding to show its gen_ai.* and ferro.* attributes" width="100%" />
|
|
283
|
+
<br/>
|
|
284
|
+
<em>Jaeger — one request's <code>gateway.request</code> span, opened to reveal its <code>gen_ai.*</code> and <code>ferro.*</code> attributes.</em>
|
|
285
|
+
</p>
|
|
286
|
+
|
|
287
|
+
Enable tracing with one variable (or the `observability:` config block —
|
|
288
|
+
endpoint, protocol, sampling, privacy, headers are all documented in the guide):
|
|
289
|
+
|
|
290
|
+
```bash
|
|
291
|
+
export OTEL_EXPORTER_OTLP_ENDPOINT=localhost:4317
|
|
292
|
+
ferrogw serve
|
|
293
|
+
```
|
|
294
|
+
|
|
295
|
+
---
|
|
296
|
+
|
|
297
|
+
## CLI
|
|
298
|
+
|
|
299
|
+
`ferrogw` is a single binary — no separate CLI tool required.
|
|
300
|
+
|
|
301
|
+
| Command | Description |
|
|
302
|
+
|:--------|:------------|
|
|
303
|
+
| `ferrogw` | Start the gateway server (default) |
|
|
304
|
+
| `ferrogw serve` | Start the gateway server (explicit) |
|
|
305
|
+
| `ferrogw init` | First-run setup — generate master key and config |
|
|
306
|
+
| `ferrogw validate` | Validate a config file without starting |
|
|
307
|
+
| `ferrogw doctor` | Check environment (API keys, config, connectivity) |
|
|
308
|
+
| `ferrogw status` | Show gateway health and provider status |
|
|
309
|
+
| `ferrogw version` | Print version, commit, and build info |
|
|
310
|
+
| `ferrogw admin keys list` | List API keys |
|
|
311
|
+
| `ferrogw admin keys create --name <name>` | Create an API key (`--scope`, `--expires-in`) |
|
|
312
|
+
| `ferrogw admin logs stats` | Show request log statistics |
|
|
313
|
+
| `ferrogw plugins` | List registered plugins |
|
|
314
|
+
|
|
315
|
+
Global flags available on all subcommands: `--gateway-url`, `--api-key`, `--format` (table/json/yaml).
|
|
316
|
+
|
|
317
|
+
---
|
|
318
|
+
|
|
319
|
+
## Deployment
|
|
320
|
+
|
|
321
|
+
### Local development
|
|
322
|
+
|
|
323
|
+
```bash
|
|
324
|
+
export OPENAI_API_KEY=sk-your-key
|
|
325
|
+
export MASTER_KEY=fgw_your-master-key
|
|
326
|
+
export GATEWAY_CONFIG=./config.yaml
|
|
327
|
+
make build && ./bin/ferrogw
|
|
328
|
+
```
|
|
329
|
+
|
|
330
|
+
### Railway & Render
|
|
331
|
+
|
|
332
|
+
The deploy buttons at the top of this README provision either platform: Railway
|
|
333
|
+
with SQLite on a volume (set the three `*_STORE_DSN` variables to paths under
|
|
334
|
+
`/data`) or PostgreSQL, and Render from the repo's `render.yaml` Blueprint,
|
|
335
|
+
which generates `MASTER_KEY` and wires the store DSNs to a managed Postgres
|
|
336
|
+
automatically.
|
|
337
|
+
|
|
338
|
+
### Docker Compose
|
|
339
|
+
|
|
340
|
+
Three Compose files in `deploy/` follow the standard override pattern — a shared
|
|
341
|
+
base, a dev override that builds from source, and a prod override with a pinned
|
|
342
|
+
tag, health check, and resource limits. Run everything from the repository root:
|
|
343
|
+
|
|
344
|
+
```bash
|
|
345
|
+
make up # dev: builds from source
|
|
346
|
+
IMAGE_TAG=v1.4.0 CORS_ORIGINS=https://your-domain.com make up-prod
|
|
347
|
+
make down # tears down either
|
|
348
|
+
```
|
|
349
|
+
|
|
350
|
+
One container serves both the API and the dashboard — no second image, no second
|
|
351
|
+
origin. Provider keys go in a repository-root `.env` or the environment.
|
|
352
|
+
[deploy/README.md](deploy/README.md) has the full reference, including a
|
|
353
|
+
self-contained PostgreSQL pairing and the fullstack observability stack.
|
|
354
|
+
|
|
355
|
+
### Kubernetes via Helm
|
|
356
|
+
|
|
357
|
+
```bash
|
|
358
|
+
helm repo add ferro-labs https://ferro-labs.github.io/helm-charts
|
|
359
|
+
helm repo update
|
|
360
|
+
helm install ferro-gw ferro-labs/ai-gateway \
|
|
361
|
+
--set env.OPENAI_API_KEY=sk-your-key
|
|
362
|
+
```
|
|
363
|
+
|
|
364
|
+
Helm charts: [github.com/ferro-labs/helm-charts](https://github.com/ferro-labs/helm-charts) | [ArtifactHub](https://artifacthub.io/packages/search?org=ferro-labs)
|
|
365
|
+
|
|
366
|
+
---
|
|
367
|
+
|
|
368
|
+
## Migrate to Ferro Labs AI Gateway
|
|
369
|
+
|
|
370
|
+
The gateway is OpenAI-compatible, so migration from any gateway — or from
|
|
371
|
+
calling a provider directly — is a `base_url` change.
|
|
372
|
+
|
|
373
|
+
### From LiteLLM
|
|
374
|
+
|
|
375
|
+
**Before (LiteLLM):**
|
|
376
|
+
|
|
377
|
+
```python
|
|
378
|
+
from litellm import completion
|
|
379
|
+
|
|
380
|
+
response = completion(
|
|
381
|
+
model="gpt-4o",
|
|
382
|
+
messages=[{"role": "user", "content": "Hello"}]
|
|
383
|
+
)
|
|
384
|
+
```
|
|
385
|
+
|
|
386
|
+
**After (Ferro Labs AI Gateway):**
|
|
387
|
+
|
|
388
|
+
```python
|
|
389
|
+
from openai import OpenAI
|
|
390
|
+
|
|
391
|
+
client = OpenAI(
|
|
392
|
+
base_url="http://localhost:8080/v1",
|
|
393
|
+
api_key="your-ferro-api-key",
|
|
394
|
+
)
|
|
395
|
+
|
|
396
|
+
response = client.chat.completions.create(
|
|
397
|
+
model="gpt-4o",
|
|
398
|
+
messages=[{"role": "user", "content": "Hello"}],
|
|
399
|
+
)
|
|
400
|
+
```
|
|
401
|
+
|
|
402
|
+
Provider API keys move to environment variables (`OPENAI_API_KEY`,
|
|
403
|
+
`ANTHROPIC_API_KEY`, …); the model list becomes `targets` + `aliases` in
|
|
404
|
+
`config.yaml`.
|
|
405
|
+
|
|
406
|
+
**Why migrate from LiteLLM:**
|
|
407
|
+
|
|
408
|
+
- 14x higher throughput at 150 concurrent users (2,447 vs 175 RPS)
|
|
409
|
+
- 23x less memory at peak load (47 MB vs 1,124 MB under streaming)
|
|
410
|
+
- Single binary — no Python environment, no pip, no virtualenv
|
|
411
|
+
- Predictable latency — p99 stays under 65 ms at 150 VU vs LiteLLM's timeouts at the same concurrency
|
|
412
|
+
|
|
413
|
+
### From Portkey
|
|
414
|
+
|
|
415
|
+
The code change is the same one line — Ferro Labs uses the standard OpenAI SDK
|
|
416
|
+
with no custom headers in self-hosted mode.
|
|
417
|
+
|
|
418
|
+
**Why migrate from Portkey:**
|
|
419
|
+
|
|
420
|
+
- Fully open source — no per-request pricing, no log limits
|
|
421
|
+
- Self-hosted — your data never leaves your infrastructure
|
|
422
|
+
- No vendor lock-in — Apache 2.0 license
|
|
423
|
+
- MCP support — Portkey self-hosted lacks native MCP
|
|
424
|
+
- FerroCloud (coming soon) for teams that want a managed service
|
|
425
|
+
|
|
426
|
+
---
|
|
427
|
+
|
|
428
|
+
## Performance
|
|
429
|
+
|
|
430
|
+
Every figure in this section comes from one run: **Ferro Labs v1.0.0, measured
|
|
431
|
+
2026-03-23**. Benchmarked against Kong OSS, Bifrost, LiteLLM, and Portkey on
|
|
432
|
+
**GCP n2-standard-8** (8 vCPU, 32 GB RAM) using a **60ms fixed-latency
|
|
433
|
+
mock upstream** — results reflect gateway overhead only. Later releases have not
|
|
434
|
+
been re-measured; reproduce against the version you intend to run using the
|
|
435
|
+
commands below.
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
| VU | RPS | p50 | p99 | Memory |
|
|
440
|
+
|---:|---:|---:|---:|---:|
|
|
441
|
+
| 50 | 813 | 61.3ms | 64.1ms | 36 MB |
|
|
442
|
+
| 150 | 2,447 | 61.2ms | 63.4ms | 47 MB |
|
|
443
|
+
| 300 | 4,890 | 61.2ms | 64.4ms | 72 MB |
|
|
444
|
+
| 500 | 8,014 | 61.5ms | 72.9ms | 89 MB |
|
|
445
|
+
| 1,000 | 13,925 | 68.1ms | 111.9ms | 135 MB |
|
|
446
|
+
|
|
447
|
+
At 1,000 VU: **13,925 RPS**, p50 overhead **8.1ms**, memory **135 MB**.
|
|
448
|
+
Against the live OpenAI API, the gateway itself adds **25 microseconds** p50 in
|
|
449
|
+
a typical plugin configuration, and **2 microseconds** with no plugins enabled.
|
|
450
|
+
|
|
451
|
+
Full methodology, raw results, and flamegraph analysis:
|
|
452
|
+
[ferro-labs/ai-gateway-performance-benchmarks](https://github.com/ferro-labs/ai-gateway-performance-benchmarks)
|
|
453
|
+
(`make setup && make bench` reproduces it).
|
|
454
|
+
|
|
455
|
+
---
|
|
456
|
+
|
|
457
|
+
## FerroCloud
|
|
458
|
+
|
|
459
|
+
FerroCloud — the managed version of Ferro Labs AI Gateway with multi-tenancy, analytics, and cost governance — is coming soon.
|
|
460
|
+
|
|
461
|
+
👉 **Join the waitlist at [ferrolabs.ai](https://ferrolabs.ai)**
|
|
462
|
+
|
|
463
|
+
---
|
|
464
|
+
|
|
465
|
+
## SDKs
|
|
466
|
+
|
|
467
|
+
Official client libraries for the Ferro Labs AI Gateway — and the standard
|
|
468
|
+
OpenAI SDK works unchanged: point `base_url` at `http://your-gateway:8080/v1`.
|
|
469
|
+
|
|
470
|
+
| SDK | Install | Repository |
|
|
471
|
+
|:----|:--------|:-----------|
|
|
472
|
+
| Python | `pip install ferrolabs` | [ferro-labs/ferrolabs-python-sdk](https://github.com/ferro-labs/ferrolabs-python-sdk) |
|
|
473
|
+
| TypeScript | `npm install ferrolabs` | [ferro-labs/ferrolabs-typescript-sdk](https://github.com/ferro-labs/ferrolabs-typescript-sdk) |
|
|
474
|
+
|
|
475
|
+
<details>
|
|
476
|
+
<summary><strong>Python</strong></summary>
|
|
477
|
+
|
|
478
|
+
```python
|
|
479
|
+
from ferrolabs import FerroClient
|
|
480
|
+
|
|
481
|
+
client = FerroClient(
|
|
482
|
+
base_url="http://localhost:8080/v1",
|
|
483
|
+
api_key="your-ferro-api-key",
|
|
484
|
+
)
|
|
485
|
+
|
|
486
|
+
response = client.chat.completions.create(
|
|
487
|
+
model="gpt-4o",
|
|
488
|
+
messages=[{"role": "user", "content": "Hello"}],
|
|
489
|
+
)
|
|
490
|
+
```
|
|
491
|
+
|
|
492
|
+
</details>
|
|
493
|
+
|
|
494
|
+
<details>
|
|
495
|
+
<summary><strong>TypeScript</strong></summary>
|
|
496
|
+
|
|
497
|
+
```typescript
|
|
498
|
+
import { FerroClient } from "ferrolabs";
|
|
499
|
+
|
|
500
|
+
const client = new FerroClient({
|
|
501
|
+
baseURL: "http://localhost:8080/v1",
|
|
502
|
+
apiKey: "your-ferro-api-key",
|
|
503
|
+
});
|
|
504
|
+
|
|
505
|
+
const response = await client.chat.completions.create({
|
|
506
|
+
model: "gpt-4o",
|
|
507
|
+
messages: [{ role: "user", content: "Hello" }],
|
|
508
|
+
});
|
|
509
|
+
```
|
|
510
|
+
|
|
511
|
+
</details>
|
|
512
|
+
|
|
513
|
+
---
|
|
514
|
+
|
|
515
|
+
## Contributing
|
|
516
|
+
|
|
517
|
+
We welcome contributions. New providers go in this OSS repo only — never in FerroCloud. See [CONTRIBUTING.md](CONTRIBUTING.md) for branch strategy, commit conventions, and PR guidelines.
|
|
518
|
+
|
|
519
|
+
---
|
|
520
|
+
|
|
521
|
+
## Community
|
|
522
|
+
|
|
523
|
+
- [GitHub Discussions](https://github.com/ferro-labs/ai-gateway/discussions)
|
|
524
|
+
- [Discord](https://discord.gg/yCAeYvJeDV)
|
|
525
|
+
- Built with Ferro Labs AI Gateway? Open a PR to add to our showcase.
|
|
526
|
+
|
|
527
|
+
---
|
|
528
|
+
|
|
529
|
+
## License
|
|
530
|
+
|
|
531
|
+
Apache 2.0 — see [LICENSE](LICENSE).
|
package/bin/ferrogw.js
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// The `bin` of the `ferrogw` npm package: find the binary that this host's
|
|
3
|
+
// platform package shipped, then hand the process over to it.
|
|
4
|
+
//
|
|
5
|
+
// Why a shim exists at all: one npm package cannot carry six platform binaries
|
|
6
|
+
// without every install paying for five it will never run. So the binaries live
|
|
7
|
+
// in `@ferro-labs-ai/gateway-<platform>` packages that carry `os`/`cpu` fields —
|
|
8
|
+
// npm installs exactly the one that matches and silently skips the rest — and
|
|
9
|
+
// this file is the single `bin` entry the user actually gets.
|
|
10
|
+
//
|
|
11
|
+
// Deliberately NOT a postinstall downloader. `--ignore-scripts` is increasingly
|
|
12
|
+
// the corporate default, an offline install has nothing to download, and a
|
|
13
|
+
// download records no integrity hash in the lockfile. Optional dependencies are
|
|
14
|
+
// integrity-checked like any other and need no script execution.
|
|
15
|
+
|
|
16
|
+
"use strict";
|
|
17
|
+
|
|
18
|
+
const path = require("node:path");
|
|
19
|
+
const { spawn } = require("node:child_process");
|
|
20
|
+
|
|
21
|
+
// npm's platform vocabulary, not Go's: process.arch says `x64` where the
|
|
22
|
+
// release archive says `amd64`, and process.platform says `win32` where it says
|
|
23
|
+
// `windows`. The package names follow npm, because `os`/`cpu` are matched
|
|
24
|
+
// against these exact values.
|
|
25
|
+
const pkg = `@ferro-labs-ai/gateway-${process.platform}-${process.arch}`;
|
|
26
|
+
const exe = process.platform === "win32" ? "ferrogw.exe" : "ferrogw";
|
|
27
|
+
|
|
28
|
+
let binary;
|
|
29
|
+
try {
|
|
30
|
+
// Resolve the platform package's manifest and walk to the binary beside it,
|
|
31
|
+
// rather than resolving the binary path directly: the POSIX binary has no
|
|
32
|
+
// extension, and leaning on require.resolve's extension search for that is a
|
|
33
|
+
// subtlety with no upside. The platform packages declare no `exports`, so
|
|
34
|
+
// `<pkg>/package.json` is always resolvable.
|
|
35
|
+
binary = path.join(require.resolve(`${pkg}/package.json`), "..", exe);
|
|
36
|
+
} catch {
|
|
37
|
+
console.error(
|
|
38
|
+
`ferrogw: no prebuilt binary for ${process.platform}-${process.arch}.\n` +
|
|
39
|
+
`\n` +
|
|
40
|
+
`The optional dependency ${pkg} is not installed. Either this platform is\n` +
|
|
41
|
+
`not one the gateway publishes binaries for, or the install ran with\n` +
|
|
42
|
+
`--omit=optional / --no-optional and skipped it.\n` +
|
|
43
|
+
`\n` +
|
|
44
|
+
`Other install paths: https://get.ferrolabs.ai`,
|
|
45
|
+
);
|
|
46
|
+
process.exit(1);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// spawn, not spawnSync. On POSIX the ideal is to replace this process with the
|
|
50
|
+
// binary outright, but Node exposes no execve, so a parent process survives for
|
|
51
|
+
// the whole life of the server and has to behave like one. spawnSync blocks the
|
|
52
|
+
// event loop, so no handler could ever run: Ctrl-C would still reach the child
|
|
53
|
+
// (a tty signals the entire foreground process group), but a `kill`, a
|
|
54
|
+
// `docker stop` or a systemd unit stop targets the parent alone, and that
|
|
55
|
+
// SIGTERM has to be relayed or the gateway never gets its graceful shutdown.
|
|
56
|
+
const child = spawn(binary, process.argv.slice(2), { stdio: "inherit" });
|
|
57
|
+
|
|
58
|
+
// SIGBREAK does not exist off Windows; Node treats an unknown signal name as an
|
|
59
|
+
// ordinary event that simply never fires, so the list needs no per-OS branch.
|
|
60
|
+
const FORWARDED = ["SIGINT", "SIGTERM", "SIGHUP", "SIGBREAK"];
|
|
61
|
+
for (const signal of FORWARDED) {
|
|
62
|
+
process.on(signal, () => {
|
|
63
|
+
// `child.killed` records only that a signal was ever SENT -- it flips true
|
|
64
|
+
// on the first kill() and stays true while the child keeps running, which
|
|
65
|
+
// is the normal case here, because the gateway catches SIGINT/SIGTERM to
|
|
66
|
+
// shut down gracefully. Guarding on it relays the first signal and swallows
|
|
67
|
+
// every one after it, so a supervisor escalating SIGINT -> SIGTERM ->
|
|
68
|
+
// SIGKILL gets only the SIGINT through and then waits out its whole timeout
|
|
69
|
+
// on a process nobody asked again. exitCode/signalCode are both null
|
|
70
|
+
// exactly while the child is running, and one of them is set the moment it
|
|
71
|
+
// is not, which is the fact this guard actually wants.
|
|
72
|
+
//
|
|
73
|
+
// Wrapped because this runs INSIDE a signal handler, where an uncaught
|
|
74
|
+
// throw takes the shim down and orphans the gateway. On Windows only
|
|
75
|
+
// SIGTERM/SIGKILL/SIGINT terminate a process, and libuv raises EINVAL for
|
|
76
|
+
// the other two names in FORWARDED -- so on the one platform where SIGHUP
|
|
77
|
+
// and SIGBREAK are reachable, relaying them is what would kill the relay.
|
|
78
|
+
if (child.exitCode === null && child.signalCode === null) {
|
|
79
|
+
try {
|
|
80
|
+
child.kill(signal);
|
|
81
|
+
} catch {
|
|
82
|
+
// Nothing useful to do: the child is alive and this platform will not
|
|
83
|
+
// deliver this signal. Staying up keeps the exit-code and re-raise
|
|
84
|
+
// paths below intact, which is the more important guarantee.
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
child.on("error", (err) => {
|
|
91
|
+
console.error(`ferrogw: cannot execute ${binary}: ${err.message}`);
|
|
92
|
+
process.exit(1);
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
child.on("close", (code, signal) => {
|
|
96
|
+
if (signal) {
|
|
97
|
+
// Re-raise so the wait status a supervisor reads says "killed by SIGTERM"
|
|
98
|
+
// rather than "exited 143" — those are different facts and only one of them
|
|
99
|
+
// is true. Handlers come off first or we would trap our own signal.
|
|
100
|
+
for (const s of FORWARDED) process.removeAllListeners(s);
|
|
101
|
+
process.kill(process.pid, signal);
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
process.exit(code ?? 1);
|
|
105
|
+
});
|
package/package.json
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "ferrogw",
|
|
3
|
+
"version": "1.4.4",
|
|
4
|
+
"description": "Ferro Labs AI Gateway — one OpenAI-compatible API in front of 30+ LLM providers, with routing, guardrails and key management.",
|
|
5
|
+
"license": "Apache-2.0",
|
|
6
|
+
"author": "Ferro Labs <hello@ferrolabs.ai>",
|
|
7
|
+
"homepage": "https://github.com/ferro-labs/ai-gateway",
|
|
8
|
+
"repository": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"url": "git+https://github.com/ferro-labs/ai-gateway.git"
|
|
11
|
+
},
|
|
12
|
+
"bugs": {
|
|
13
|
+
"url": "https://github.com/ferro-labs/ai-gateway/issues"
|
|
14
|
+
},
|
|
15
|
+
"keywords": [
|
|
16
|
+
"ai",
|
|
17
|
+
"llm",
|
|
18
|
+
"gateway",
|
|
19
|
+
"openai",
|
|
20
|
+
"anthropic",
|
|
21
|
+
"proxy",
|
|
22
|
+
"routing",
|
|
23
|
+
"ferro",
|
|
24
|
+
"ferrolabs",
|
|
25
|
+
"ai-gateway"
|
|
26
|
+
],
|
|
27
|
+
"bin": {
|
|
28
|
+
"ferrogw": "bin/ferrogw.js"
|
|
29
|
+
},
|
|
30
|
+
"files": [
|
|
31
|
+
"bin"
|
|
32
|
+
],
|
|
33
|
+
"engines": {
|
|
34
|
+
"node": ">=18"
|
|
35
|
+
},
|
|
36
|
+
"optionalDependencies": {
|
|
37
|
+
"@ferro-labs-ai/gateway-linux-x64": "1.4.4",
|
|
38
|
+
"@ferro-labs-ai/gateway-linux-arm64": "1.4.4",
|
|
39
|
+
"@ferro-labs-ai/gateway-darwin-x64": "1.4.4",
|
|
40
|
+
"@ferro-labs-ai/gateway-darwin-arm64": "1.4.4",
|
|
41
|
+
"@ferro-labs-ai/gateway-win32-x64": "1.4.4",
|
|
42
|
+
"@ferro-labs-ai/gateway-win32-arm64": "1.4.4"
|
|
43
|
+
}
|
|
44
|
+
}
|