@octocodeai/freellama-mcp-server 0.6.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONFIGURATION.md +5 -7
- package/DEVELOPING.md +18 -15
- package/LICENSE-MIT +1 -1
- package/README.md +8 -7
- package/dist/index.js +7 -0
- package/docs/ARCHITECTURE.md +38 -27
- package/docs/CLI.md +15 -12
- package/docs/CPU_GPU_ROUTING.md +11 -11
- package/docs/MODEL_SELECTION.md +3 -2
- package/docs/MONITORING.md +8 -6
- package/docs/PRODUCTION.md +27 -17
- package/docs/README.md +2 -6
- package/docs/TESTING.md +5 -6
- package/package.json +10 -9
package/CONFIGURATION.md
CHANGED
|
@@ -9,7 +9,7 @@ server-launch config (for example `.mcp.json`'s `env` block) or the launching sh
|
|
|
9
9
|
|
|
10
10
|
| Variable | Default | Affects |
|
|
11
11
|
|---|---|---|
|
|
12
|
-
| `FREELLAMA_OLLAMA_ENDPOINT` | `http://127.0.0.1:11434` | Default Ollama endpoint for `doctor`, the `models` detail/raw/library views,
|
|
12
|
+
| `FREELLAMA_OLLAMA_ENDPOINT` | `http://127.0.0.1:11434` | Default Ollama endpoint for `doctor`, the `models` detail/raw/library views, all `ollama_*` tools, and the autostarted serve's `--upstream` |
|
|
13
13
|
| `FREELLAMA_SERVE_ENDPOINT` | `http://127.0.0.1:11435` | Default `endpoint` for serve-backed tools and `delegate_research` |
|
|
14
14
|
| `FREELLAMA_MCP_DEFAULT_MODEL` | unset (routes to an installed coding model) | `delegate_research`'s default `model` |
|
|
15
15
|
| `FREELLAMA_MCP_DEFAULT_ADAPTER` | `bash` | `delegate_research`'s adapter (`octocode` to switch) |
|
|
@@ -22,11 +22,11 @@ server-launch config (for example `.mcp.json`'s `env` block) or the launching sh
|
|
|
22
22
|
| `FREELLAMA_EXTERNAL_COST_MODEL` | unset | External-model label in estimated avoided-cost telemetry; set all three cost variables or none |
|
|
23
23
|
| `FREELLAMA_EXTERNAL_COST_INPUT_USD_PER_M` | unset | External input USD per million tokens |
|
|
24
24
|
| `FREELLAMA_EXTERNAL_COST_OUTPUT_USD_PER_M` | unset | External output USD per million tokens |
|
|
25
|
-
| `FREELLAMA_CONTROL_TIMEOUT_SECONDS` | `30` | Decision-only serve calls; read by `packages/rust-core/src/
|
|
26
|
-
| `FREELLAMA_TASK_TIMEOUT_SECONDS` | `900` | Generation calls; read by `packages/rust-core/src/
|
|
25
|
+
| `FREELLAMA_CONTROL_TIMEOUT_SECONDS` | `30` | Decision-only serve calls; read by the native client (`packages/rust-core/src/serve_client.rs`) |
|
|
26
|
+
| `FREELLAMA_TASK_TIMEOUT_SECONDS` | `900` | Generation calls; read by the native client (`packages/rust-core/src/serve_client.rs`) |
|
|
27
27
|
| `FREELLAMA_AUTH_TOKEN_FILE` | unset | Bearer-token file used by the native client and research adapters for all serve routes |
|
|
28
28
|
| `FREELLAMA_AGENT_TOKEN_CALIBRATION_DIR` | platform data directory | Prompt-free, model-specific token-estimator calibration shared across adapter processes |
|
|
29
|
-
| `FREELLAMA_MCP_AUTOSTART_SERVE` | `1` | When the default loopback serve endpoint does not answer, start `freellama serve` as a child of the MCP server (stopped with it). `0` disables |
|
|
29
|
+
| `FREELLAMA_MCP_AUTOSTART_SERVE` | `1` | When the default loopback serve endpoint does not answer, start `freellama serve` as a child of the MCP server (stopped with it): once in the background at launch, so other local clients find it, and again on a tool call if it is still down. `0` disables |
|
|
30
30
|
| `FREELLAMA_SERVE_BINARY` | checkout `target/release`/`debug`, then the platform package | Binary used for serve autostart |
|
|
31
31
|
| `FREELLAMA_AGENT_OS_SANDBOX` | `auto` | Bash adapter OS confinement: `bwrap` on Linux, `sandbox-exec` on macOS, when present. `off` keeps only the restricted shell |
|
|
32
32
|
| `FREELLAMA_AGENT_OCTOCODE_PACKAGE` | `octocode@19.1.0` | Pinned npm spec the Octocode adapter runs through `npx` |
|
|
@@ -103,6 +103,4 @@ slots or relax memory guards. Existing queued work keeps its captured cost. See
|
|
|
103
103
|
`packages/mcp/evidence/model-evidence.json`. A published package does not include that repository
|
|
104
104
|
file, so its evidence table is empty unless you set `FREELLAMA_MCP_MODEL_EVIDENCE`. An unmeasured
|
|
105
105
|
model yields a `verify` verdict, and an unreadable evidence file is reported on stderr and treated
|
|
106
|
-
as empty. Tool descriptions state the rules
|
|
107
|
-
[`skills/freellama/references/`](https://github.com/bgauryy/FreeLlama/tree/main/skills/freellama/references/) and
|
|
108
|
-
[`docs/dev/evidence/`](https://github.com/bgauryy/FreeLlama/tree/main/docs/dev/evidence/).
|
|
106
|
+
as empty. Tool descriptions state the rules.
|
package/DEVELOPING.md
CHANGED
|
@@ -13,10 +13,11 @@ yarn install
|
|
|
13
13
|
yarn build
|
|
14
14
|
```
|
|
15
15
|
|
|
16
|
-
`yarn build` writes
|
|
16
|
+
`yarn build` writes the MCP server, `packages/mcp/dist/index.js` (shebang, executable). During
|
|
17
17
|
checkout development it loads `native/freellama.<triple>.node`. Published installs resolve the
|
|
18
18
|
matching `@octocodeai/freellama-native-<target>` optional package instead, so the base MCP package remains
|
|
19
|
-
portable.
|
|
19
|
+
portable. The research adapters compile to `dist/research/{bash-agent,octocode-agent}.js`, and the
|
|
20
|
+
repository `docs/` guides are copied into `docs/`.
|
|
20
21
|
|
|
21
22
|
Piecewise, if you are iterating on one layer:
|
|
22
23
|
|
|
@@ -24,7 +25,7 @@ Piecewise, if you are iterating on one layer:
|
|
|
24
25
|
# 1. Native addon → packages/mcp/native/freellama.<triple>.node
|
|
25
26
|
yarn build:native
|
|
26
27
|
|
|
27
|
-
# 2. This server → dist/index.js via esbuild
|
|
28
|
+
# 2. This server → dist/index.js and dist/research/*.js via esbuild, plus docs/
|
|
28
29
|
yarn workspace @octocodeai/freellama-mcp-server build
|
|
29
30
|
```
|
|
30
31
|
|
|
@@ -33,8 +34,8 @@ root produces both in a checkout. `dist/index.js` loads `native/index.js`, which
|
|
|
33
34
|
first and then a matching optional platform package. The `.node` binary is gitignored (a build
|
|
34
35
|
artifact). `native/index.js` is a small hand-written loader; `native/index.d.ts` is generated from
|
|
35
36
|
`napi.rs` by `yarn build:native`. Both are checked in; do not edit the declarations by hand —
|
|
36
|
-
change the Rust doc comments or `ts_args_type`, rebuild, and commit.
|
|
37
|
-
|
|
37
|
+
change the Rust doc comments or `ts_args_type`, rebuild, and commit. The unit test
|
|
38
|
+
`test/unit/native-dts-drift.test.ts` fails when the declarations and the `napi.rs` exports differ.
|
|
38
39
|
|
|
39
40
|
## Test
|
|
40
41
|
|
|
@@ -42,15 +43,17 @@ Vitest, in three tiers (all TypeScript, under `test/`):
|
|
|
42
43
|
|
|
43
44
|
| Tier | Command (from this package) | Checks | Needs |
|
|
44
45
|
|---|---|---|---|
|
|
45
|
-
| `test/unit/` | `yarn test` (watch: `yarn test:watch`) | pure functions straight from `src/*.ts` — no build step, this is the TDD loop | nothing,
|
|
46
|
+
| `test/unit/` | `yarn test` (watch: `yarn test:watch`) | pure functions straight from `src/*.ts` — no build step, this is the TDD loop | nothing, a few seconds |
|
|
46
47
|
| `test/integration/` | `yarn test:integration` | the built server over the real MCP protocol: tool contract, structured content, guardrails; rebuilds `dist/` itself | Ollama at `FREELLAMA_OLLAMA_ENDPOINT` (default :11434) |
|
|
47
48
|
| `test/e2e/` | `yarn test:e2e` | every tool against the live system: real routing/execution, pull → delete round trip (net-zero), a real delegated research run | serve + Ollama + models, ~40s |
|
|
48
49
|
|
|
49
|
-
The same commands work from the repository root (`yarn test` there also runs the CLI package's tests
|
|
50
|
+
The same commands work from the repository root (`yarn test` there also runs the CLI package's tests
|
|
51
|
+
and the Rust suite).
|
|
50
52
|
The integration/e2e tiers fail fast with a readable message when Ollama is down, and the lifecycle
|
|
51
53
|
test refuses to delete a model already installed on the test system.
|
|
52
54
|
|
|
53
|
-
For default-endpoint autostart,
|
|
55
|
+
For default-endpoint autostart, the server starts `freellama serve` in the background at launch and
|
|
56
|
+
logs a failure to stderr. On a tool call, a spawn failure, early exit, or readiness timeout rejects the
|
|
54
57
|
request while leaving the MCP transport available for diagnosis or retry. Autostart does not apply
|
|
55
58
|
to an explicit non-default endpoint; see `FREELLAMA_MCP_AUTOSTART_SERVE` in
|
|
56
59
|
[configuration](CONFIGURATION.md#environment-variables).
|
|
@@ -74,8 +77,8 @@ Start the serve-backed tools' dependency from the repository root:
|
|
|
74
77
|
cargo run --release -- serve
|
|
75
78
|
```
|
|
76
79
|
|
|
77
|
-
|
|
78
|
-
|
|
80
|
+
`freellama proxy` only forwards raw `/api/*` calls; only `serve` has the `/_freellama/v1/*` routes
|
|
81
|
+
these tools call.
|
|
79
82
|
|
|
80
83
|
## Publish
|
|
81
84
|
|
|
@@ -91,18 +94,18 @@ cd packages/mcp
|
|
|
91
94
|
npm pack --dry-run # confirm exactly what a publish would ship
|
|
92
95
|
```
|
|
93
96
|
|
|
94
|
-
Treat `npm publish` as a public release action. Follow the
|
|
95
|
-
[release
|
|
97
|
+
Treat `npm publish` as a public release action. Follow the
|
|
98
|
+
[release package checks](https://github.com/bgauryy/FreeLlama/blob/main/docs/TESTING.md#verify-release-packages) and obtain the repository owner's release authorization first.
|
|
96
99
|
|
|
97
100
|
## Why native bindings
|
|
98
101
|
|
|
99
102
|
- **No routing reimplementation.** `run_task`, `run_task {preview:true}`, and the installed-model list are HTTP
|
|
100
|
-
calls into a running `freellama serve`, matching the CLI
|
|
103
|
+
calls into a running `freellama serve`, matching the CLI requests in `packages/cli/src/requests.rs`. `doctor` talks to Ollama
|
|
101
104
|
and the host's portable machine-discovery layer directly. `ollama_manage` / `ollama_delete` hit Ollama's HTTP API. `delegate_research`
|
|
102
105
|
is a subprocess. Routing decisions still live in exactly one place — the Rust core; see
|
|
103
106
|
[what the MCP layer computes](README.md#what-the-mcp-layer-computes).
|
|
104
|
-
- **One connection pool.** `packages/rust-core/src/
|
|
105
|
-
`OnceLock` and clones the handle per request
|
|
107
|
+
- **One connection pool.** `packages/rust-core/src/serve_client.rs` holds a single `reqwest::Client` in a
|
|
108
|
+
`OnceLock` and clones the handle per request; `napi.rs` and the CLI share it.
|
|
106
109
|
- **`unsafe_code` stays isolated.** The crate is `deny(unsafe_code)` except `napi.rs`, which
|
|
107
110
|
carries an explicit `#[allow(unsafe_code)]` for napi-derive's FFI glue.
|
|
108
111
|
- **The napi build is off by default** (`cargo build` never touches it); the addon is built with
|
package/LICENSE-MIT
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
MIT License
|
|
2
2
|
|
|
3
|
-
Copyright (c)
|
|
3
|
+
Copyright (c) 2026 Guy Bary
|
|
4
4
|
|
|
5
5
|
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
|
|
6
6
|
associated documentation files (the "Software"), to deal in the Software without restriction, including
|
package/README.md
CHANGED
|
@@ -14,7 +14,7 @@ The server uses the official [TypeScript SDK](https://github.com/modelcontextpro
|
|
|
14
14
|
|
|
15
15
|
## Understand the tools
|
|
16
16
|
|
|
17
|
-
`doctor`, `models`, `run_task`, and `
|
|
17
|
+
`doctor`, `models`, `run_task`, `task_jobs`, `scope`, and `session` call the Rust core through native NAPI bindings
|
|
18
18
|
(`packages/rust-core/src/napi.rs`), with no CLI subprocess and no routing decisions in TypeScript; the
|
|
19
19
|
TypeScript layer validates request shapes and derives presentation fields (see
|
|
20
20
|
[what the MCP layer computes](#what-the-mcp-layer-computes)). `run_task` has four modes: one task,
|
|
@@ -218,9 +218,8 @@ The TypeScript server adds a small set of derived fields on top of core results:
|
|
|
218
218
|
|
|
219
219
|
## Read packaged documentation through MCP
|
|
220
220
|
|
|
221
|
-
The published MCP package includes the
|
|
222
|
-
directory
|
|
223
|
-
source of truth; the package build copies it into `docs/`.
|
|
221
|
+
The published MCP package includes the Markdown guides from the repository `docs/` directory.
|
|
222
|
+
The repository directory remains the source of truth; the package build copies it into `docs/`.
|
|
224
223
|
|
|
225
224
|
MCP clients can list resources and read `freellama://docs/index` (the repository `docs/README.md`),
|
|
226
225
|
then fetch exactly one relevant guide such as `freellama://docs/PRODUCTION` or
|
|
@@ -243,6 +242,8 @@ node packages/mcp/dist/index.js
|
|
|
243
242
|
```
|
|
244
243
|
|
|
245
244
|
The server speaks MCP over stdio — configure it in your MCP client, do not run it interactively.
|
|
245
|
+
At launch it starts `freellama serve` in the background when the default endpoint does not answer;
|
|
246
|
+
see `FREELLAMA_MCP_AUTOSTART_SERVE` in [Configuration](CONFIGURATION.md#environment-variables).
|
|
246
247
|
|
|
247
248
|
**Claude Desktop (`claude_desktop_config.json`):**
|
|
248
249
|
|
|
@@ -318,8 +319,8 @@ Memory admission, authentication, and capability requirements apply in every mod
|
|
|
318
319
|
tasks or `input` for embeddings, plus any applicable Ollama runtime controls.
|
|
319
320
|
|
|
320
321
|
Preview rejects `prompt`, `systemPrompt`, `messages`, `input`, `images`, `tools`, `keepAlive`, `format`, `think`,
|
|
321
|
-
`options`, `logprobs`, `topLogprobs`, `returnEmbeddings`, `priority`, `maxWaitSeconds`, `timeoutSeconds`,
|
|
322
|
-
`defer`, `scopeId`, and `scopeRevision`. This prevents a client from attaching
|
|
322
|
+
`options`, `logprobs`, `topLogprobs`, `returnEmbeddings`, `priority`, `maxWaitSeconds`, `timeoutSeconds`,
|
|
323
|
+
`stopOnRepetition`, `defer`, `scopeId`, and `scopeRevision`. This prevents a client from attaching
|
|
323
324
|
work to a decision-only call and mistakenly assuming it ran.
|
|
324
325
|
|
|
325
326
|
`run_task.messages` preserves Ollama message fields beyond `role` and `content`, including images,
|
|
@@ -368,7 +369,7 @@ Scripts that bypass MCP post the same fields in snake_case to `POST /_freellama/
|
|
|
368
369
|
`task`, `objective`, `model`, `session_id`, `required_capabilities`, `prompt` or `messages`,
|
|
369
370
|
`images`, `priority`, `keep_alive`, `max_wait_seconds`, `timeout_seconds`, `stop_on_repetition`,
|
|
370
371
|
and `request_options: {options, format, think, logprobs, top_logprobs}`. A batch is
|
|
371
|
-
`{tasks: [{id, independent: true, task: {...}}], max_parallelism}`; sessions are
|
|
372
|
+
`POST /_freellama/v1/task-batches` with `{tasks: [{id, independent: true, task: {...}}], max_parallelism}`; sessions are
|
|
372
373
|
`POST /_freellama/v1/sessions` with `{}`. `model` accepts a bare name (`name` resolves to
|
|
373
374
|
`name:latest`, or the one installed tag of that family); an ambiguous or unknown name is refused
|
|
374
375
|
with the installed candidates. See [Request shapes](https://github.com/bgauryy/FreeLlama/blob/main/docs/ARCHITECTURE.md#request-shapes)
|
package/dist/index.js
CHANGED
|
@@ -21629,6 +21629,12 @@ function serveBinaryCandidates() {
|
|
|
21629
21629
|
if (process.env.FREELLAMA_SERVE_BINARY) candidates.unshift(process.env.FREELLAMA_SERVE_BINARY);
|
|
21630
21630
|
return candidates;
|
|
21631
21631
|
}
|
|
21632
|
+
function startServeInBackground() {
|
|
21633
|
+
ensureServe(void 0).catch((error2) => {
|
|
21634
|
+
process.stderr.write(`[freellama serve] background start failed: ${error2 instanceof Error ? error2.message : String(error2)}
|
|
21635
|
+
`);
|
|
21636
|
+
});
|
|
21637
|
+
}
|
|
21632
21638
|
function stopAutostartedServe() {
|
|
21633
21639
|
if (child && child.exitCode === null) child.kill("SIGTERM");
|
|
21634
21640
|
child = null;
|
|
@@ -23942,3 +23948,4 @@ registerOllamaLifecycleTools(server);
|
|
|
23942
23948
|
registerDelegateResearchTool(server);
|
|
23943
23949
|
var transport = withoutSchemaDialect(new StdioServerTransport());
|
|
23944
23950
|
await server.connect(transport);
|
|
23951
|
+
startServeInBackground();
|
package/docs/ARCHITECTURE.md
CHANGED
|
@@ -27,7 +27,7 @@ This is the canonical ownership rule; other guides link here.
|
|
|
27
27
|
| Forecast memory reservations, host-pressure holds, and requests to unload idle unpinned models | FreeLlama managed execution |
|
|
28
28
|
| Explicit per-model CPU/GPU backend assignment | FreeLlama and separate Ollama processes |
|
|
29
29
|
| Grounded research adapters, citations, and verification verdicts | FreeLlama MCP server |
|
|
30
|
-
| Bearer authentication of every route | FreeLlama; external ingress owns TLS and tenant authorization |
|
|
30
|
+
| Bearer authentication of every route except the static `/_freellama/ui` shell | FreeLlama; external ingress owns TLS and tenant authorization |
|
|
31
31
|
|
|
32
32
|
FreeLlama is a governed router, not an unrestricted hardware scheduler: a CPU preference cannot move
|
|
33
33
|
an arbitrary tag, and one managed call contains one task. Only the calling agent knows which tasks
|
|
@@ -92,9 +92,10 @@ flowchart TB
|
|
|
92
92
|
ADAPTER --> GPU
|
|
93
93
|
```
|
|
94
94
|
|
|
95
|
-
The CLI and MCP server do not reimplement routing.
|
|
96
|
-
|
|
97
|
-
|
|
95
|
+
The CLI and MCP server do not reimplement routing. CLI commands other than `serve`, `doctor`,
|
|
96
|
+
`init`, and benchmarking are HTTP clients of a running `freellama serve`. The MCP server reaches the
|
|
97
|
+
same control API through the feature-gated NAPI boundary, and unless `FREELLAMA_MCP_AUTOSTART_SERVE=0`
|
|
98
|
+
it starts the bundled `serve` in the background at launch when nothing answers on the default endpoint.
|
|
98
99
|
|
|
99
100
|
## Request classification
|
|
100
101
|
|
|
@@ -124,17 +125,21 @@ The control API exposes these endpoints:
|
|
|
124
125
|
| `POST /_freellama/v1/routes` | Deterministic route decision | No |
|
|
125
126
|
| `POST /_freellama/v1/sessions` | New model-affinity session | No |
|
|
126
127
|
| `DELETE /_freellama/v1/sessions/:session_id` | Release model-affinity session | No |
|
|
128
|
+
| `POST /_freellama/v1/sessions/:session_id/kill` | Release a session and cancel its in-flight tasks; the runner stays loaded | No new inference |
|
|
127
129
|
| `POST /_freellama/v1/scopes` | Create bounded message history | No |
|
|
128
130
|
| `GET /_freellama/v1/scopes/:id` | Scope metadata; optional explicit history inclusion | No |
|
|
129
131
|
| `POST /_freellama/v1/scopes/:id/fork` | Fork history at an expected revision | No |
|
|
130
132
|
| `DELETE /_freellama/v1/scopes/:id` | Delete retained history | No |
|
|
131
133
|
| `POST /_freellama/v1/warm` | Admitted load of an exact installed runner profile | Load only |
|
|
132
134
|
| `POST /_freellama/v1/tasks` | Managed chat, vision, tools, or embedding task (body shape under [Request shapes](#request-shapes)) | Yes |
|
|
135
|
+
| `POST /_freellama/v1/task-batches` | Caller-declared independent tasks with ordered per-item receipts | Yes |
|
|
133
136
|
| `GET /_freellama/v1/jobs` | Deferred job metadata without prompts or results | No |
|
|
134
137
|
| `GET /_freellama/v1/jobs/:id` | One deferred receipt and retained result | No |
|
|
135
138
|
| `POST /_freellama/v1/jobs/:id/cancel` | Cancel one deferred task and release its local permits | No new inference |
|
|
136
139
|
| `DELETE /_freellama/v1/jobs/:id` | Cancel active work, release local permits, and remove its retained record | No new inference |
|
|
137
140
|
|
|
141
|
+
Status, usage, metrics, and runtime-config endpoints are listed in [monitoring](MONITORING.md).
|
|
142
|
+
|
|
138
143
|
### Request shapes
|
|
139
144
|
|
|
140
145
|
The control API uses snake_case JSON. A managed task body is the routing fields plus the payload:
|
|
@@ -173,7 +178,7 @@ A batch nests whole task bodies:
|
|
|
173
178
|
`POST /sessions` with an empty object returns `session_id`; `POST /warm` takes `{"model": "<tag>"}`.
|
|
174
179
|
|
|
175
180
|
The machine profile is host-derived rather than model-name-derived. It reports total physical
|
|
176
|
-
memory on macOS (`sysctl`), Linux (`/proc`), and Windows (
|
|
181
|
+
memory on macOS (`sysctl`), Linux (`/proc`), and Windows (CIM through PowerShell), along with CPU, OS,
|
|
177
182
|
architecture, and disk. `memory_bytes` is portable host RAM. `unified_memory_bytes` is populated
|
|
178
183
|
only when host and accelerator memory are known to share a pool; it must not be treated as discrete
|
|
179
184
|
GPU VRAM. Ollama's `/api/ps` remains the placement authority.
|
|
@@ -208,7 +213,7 @@ Every field comes from the model's own metadata and from what FreeLlama observed
|
|
|
208
213
|
name. `GET /_freellama/v1/models` adds, per model:
|
|
209
214
|
|
|
210
215
|
- `context_default`: the context Ollama gives a request that leaves `num_ctx` unset (`source` is
|
|
211
|
-
`modelfile`, `config`, `ollama_context_length`, `vram_tier`, or `
|
|
216
|
+
`modelfile`, `config`, `ollama_context_length`, `vram_tier`, `resident_runner`, or `unknown`). A managed
|
|
212
217
|
task that sends a different `num_ctx` reloads the runner.
|
|
213
218
|
- `resident_context`: the `context_length` of the loaded runner from `/api/ps`.
|
|
214
219
|
- `observed_loops`: per task kind, managed runs since start and how many ended in a repetition
|
|
@@ -243,7 +248,7 @@ flowchart TD
|
|
|
243
248
|
F --> X{"Explicit model?"}
|
|
244
249
|
X -->|"Yes"| E["Use exact model or refuse"]
|
|
245
250
|
X -->|"No"| O{"Objective"}
|
|
246
|
-
O -->|"fastest"| S["Rank local
|
|
251
|
+
O -->|"fastest"| S["Rank local benchmark,<br/>smaller size, and deterministic ties"]
|
|
247
252
|
O -->|"balanced"| B["Prefer policy-qualified candidates;<br/>otherwise use eligible models with low confidence"]
|
|
248
253
|
O -->|"quality"| Q["Require policy-qualified candidates,<br/>then prefer policy order"]
|
|
249
254
|
E --> H["Grade advertised-context compatibility and evidence"]
|
|
@@ -264,8 +269,9 @@ Routing never ranks by a model's name or architecture. Automatic selection (fall
|
|
|
264
269
|
session affinity) ranks a model last for a task kind when it has looped on that kind at least twice
|
|
265
270
|
in at most twice as many managed runs; the decision then carries
|
|
266
271
|
`repeats_on_this_task_last_resort`. An operator policy naming the model for the task, or an explicit
|
|
267
|
-
`model`, always wins. No request option is
|
|
268
|
-
and decoding settings belong to
|
|
272
|
+
`model`, always wins. No request option is hard-coded for a model: apart from an operator policy's
|
|
273
|
+
per-model `model_options` (`think`, `num_predict`), stop sequences and decoding settings belong to
|
|
274
|
+
the caller.
|
|
269
275
|
|
|
270
276
|
`context_window_fit` means only that requested `num_ctx` fits the model's advertised context
|
|
271
277
|
window. It is not a live-memory, free-VRAM, K/V-cache, thermal, or runner-admission verdict.
|
|
@@ -306,7 +312,7 @@ flowchart TD
|
|
|
306
312
|
T["POST /tasks or /task-batches"] --> R["Select eligible model + backend"]
|
|
307
313
|
R --> K["Estimate metadata-backed F16 K/V<br/>(or report unknown)"]
|
|
308
314
|
K --> A{"Acquire selected backend's<br/>3:2:1 fair weighted budget"}
|
|
309
|
-
A -->|"No"| E503["503
|
|
315
|
+
A -->|"No"| E503["429 queue full or<br/>503 wait timeout"]
|
|
310
316
|
A -->|"Yes"| B{"Execution backend"}
|
|
311
317
|
B -->|"Primary"| GL{"GPU model resident<br/>under transition lock?"}
|
|
312
318
|
B -->|"CPU assignment"| CO["Add num_gpu: 0"]
|
|
@@ -334,8 +340,8 @@ falling back to two. The optional CPU pool defaults to one. A saturated GPU burs
|
|
|
334
340
|
consume the permit held for a small CPU helper. With one parallel slot, those values represent one
|
|
335
341
|
ordinary chat and one embedding. Operators can tune both budgets from queue-wait and resident-memory evidence.
|
|
336
342
|
After FreeLlama admission, each Ollama process applies its own `OLLAMA_MAX_QUEUE`, scheduler,
|
|
337
|
-
`OLLAMA_NUM_PARALLEL`, and loaded-model limit. Raw proxy requests bypass weighted
|
|
338
|
-
still pass the raw concurrency cap, backend exclusion, and host-pressure gate.
|
|
343
|
+
`OLLAMA_NUM_PARALLEL`, and loaded-model limit. Raw proxy generation requests bypass weighted
|
|
344
|
+
admission but still pass the raw concurrency cap, backend exclusion, and host-pressure gate.
|
|
339
345
|
|
|
340
346
|
| Task | Default cost |
|
|
341
347
|
|---|---:|
|
|
@@ -375,9 +381,10 @@ stable ID and `independent:true`; dependent workflow execution is rejected befor
|
|
|
375
381
|
It bounds local dispatch and returns ordered per-item success/error receipts. Every child still
|
|
376
382
|
passes the same global admission and transition path above.
|
|
377
383
|
|
|
378
|
-
The K/V preflight derives single-sequence F16 K+V bytes-per-token
|
|
379
|
-
|
|
380
|
-
|
|
384
|
+
The K/V preflight derives single-sequence F16 K+V bytes-per-token for any architecture whose
|
|
385
|
+
metadata declares the block count and head counts. Explicit key/value dimensions take precedence
|
|
386
|
+
over embedding length divided by heads; sliding-window, shared-KV, recurrent, cross-attention, and
|
|
387
|
+
unknown layouts report unknown. F16 is advisory because quantized KV can
|
|
381
388
|
use less memory. The coarse model-file budget refuses files over 80% of local CPU/unified RAM only
|
|
382
389
|
for loopback backends. Forecasts use each backend's independently attributed parallelism and KV
|
|
383
390
|
cache settings where process inspection is available, with explicit fallback sources otherwise.
|
|
@@ -393,9 +400,10 @@ growth holds new loads; CPU-load and thermal holds apply to CPU-assigned work an
|
|
|
393
400
|
already resident on the GPU, which neither waits for the cores nor cools the package by waiting (`ResourceDemand::is_held_by`; the assessment's
|
|
394
401
|
`held_by` lists the holds that applied). Low available RAM (below the larger of 15% or 1 GiB, clearing after two
|
|
395
402
|
healthy samples above the larger of 20% or 2 GiB) depends on the platform (`ResourcePolicy::low_memory`).
|
|
403
|
+
A new load must leave that reserve free, but never more than its own size above the 1 GiB floor.
|
|
396
404
|
On macOS (`advisory`) the estimate undercounts what the OS can reclaim, so a load that does not cover
|
|
397
|
-
it first unloads idle runners and then dispatches with a `memory_advisory`. On Linux
|
|
398
|
-
|
|
405
|
+
it first unloads idle runners and then dispatches with a `memory_advisory`. On Linux and other
|
|
406
|
+
platforms (`refuse`) a new load that does not fit first unloads idle runners and then waits
|
|
399
407
|
with `held_insufficient_capacity` until a permit release or a fresh sample frees capacity, or refuses
|
|
400
408
|
with 503 at its deadline: Linux `MemAvailable` already includes reclaimable cache and is the hard cap
|
|
401
409
|
in a container, so over-admitting invites the OOM killer. Already-resident runners add no memory and
|
|
@@ -406,8 +414,10 @@ seconds; unavailable signals stay unknown. Remote backends bypass this host gate
|
|
|
406
414
|
Each pressure signal recovers independently after two fresh samples that meet its recovery
|
|
407
415
|
threshold. A recovered CPU-load signal therefore clears even while RAM remains low.
|
|
408
416
|
|
|
409
|
-
|
|
410
|
-
even when a raw request has no model-footprint estimate.
|
|
417
|
+
On Linux and macOS the default `require_memory` telemetry policy holds local inference when
|
|
418
|
+
available RAM is missing, even when a raw request has no model-footprint estimate. Windows and other
|
|
419
|
+
platforms have no host collector, so they default to `best_effort` and admit with
|
|
420
|
+
`admitted_telemetry_unknown`; elsewhere `best_effort` is an explicit opt-in;
|
|
411
421
|
`require_all` additionally requires the collector's applicable load, OS-pressure, and thermal signals.
|
|
412
422
|
Unsupported Linux collector signals remain explicitly unavailable rather than being fabricated.
|
|
413
423
|
The pure `ResourceSnapshot::assess_capacity` calculation is shared by admission and previews.
|
|
@@ -442,11 +452,11 @@ Resident tasks share a backend lock. Nonresident tasks take that backend's exclu
|
|
|
442
452
|
load cannot race another managed task on the same server. CPU and GPU backends have separate locks
|
|
443
453
|
and admission pools, so they can progress independently.
|
|
444
454
|
|
|
445
|
-
In `serve`,
|
|
446
|
-
returning 503 when busy. The response body holds this guard through
|
|
447
|
-
GET/HEAD
|
|
448
|
-
|
|
449
|
-
pressure
|
|
455
|
+
In `serve`, raw generation requests also take the primary backend's exclusive transition lock,
|
|
456
|
+
returning 503 when it stays busy past the raw queue wait. The response body holds this guard through
|
|
457
|
+
EOF or disconnect. Metadata GET/HEAD, `/api/show`, model-store calls (pull, push, copy, delete, and
|
|
458
|
+
blob uploads), and empty `keep_alive:0` unload requests take no lock and bypass pressure. Raw
|
|
459
|
+
inference waits at most two seconds for host pressure to clear, then returns 503 with `Retry-After`. Raw request-body collection has the configured request deadline too; an incomplete upload
|
|
450
460
|
returns HTTP 408 and releases its guards without contacting Ollama. Admission receipts report
|
|
451
461
|
`queue_wait_ms`, `resource_wait_ms`, `transition_wait_ms`, and
|
|
452
462
|
their sum as `total_wait_ms`. Health and route receipts expose sampled pressure and reservations.
|
|
@@ -599,7 +609,7 @@ expected revision. Warming uses ordinary admission and fit checks; omitted resid
|
|
|
599
609
|
a finite measured policy. See [Scope history and model warming](SCOPES_AND_WARMING.md) for the
|
|
600
610
|
public contract and configured limits.
|
|
601
611
|
|
|
602
|
-
In `serve`,
|
|
612
|
+
In `serve`, raw generation traffic shares the primary transition lock and receives
|
|
603
613
|
host-pressure gating, while managed tasks also reserve forecast memory. Direct Ollama callers bypass
|
|
604
614
|
these controls. FreeLlama cannot preempt inference or constrain other processes. A configured but
|
|
605
615
|
unreachable CPU backend makes managed catalog discovery fail closed; raw primary passthrough remains
|
|
@@ -614,8 +624,9 @@ explicit model that is not installed, or a task that exceeds the advertised cont
|
|
|
614
624
|
## Product boundary
|
|
615
625
|
|
|
616
626
|
FreeLlama is not an inference engine, remote provider marketplace, billing layer, hosted model
|
|
617
|
-
registry, general agent runtime, or A2A coordinator. It can authenticate every route
|
|
618
|
-
operator-managed bearer token and refuses unauthenticated nonloopback
|
|
627
|
+
registry, general agent runtime, or A2A coordinator. It can authenticate every route except the
|
|
628
|
+
static UI shell with one operator-managed bearer token and refuses unauthenticated nonloopback
|
|
629
|
+
listeners. An external layer
|
|
619
630
|
must still provide TLS, tenant authorization, tenant-specific limits, and isolation.
|
|
620
631
|
|
|
621
632
|
Use Ollama directly when a caller only needs raw speed from one exact model. Use FreeLlama when the
|
package/docs/CLI.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
The `freellama` command starts and inspects the local management layer that agents use to offload
|
|
4
4
|
work. It exposes diagnostics, model routing, managed execution, resource controls, benchmarking,
|
|
5
|
-
and Ollama-compatible passthrough. Use the [README CLI section](https://github.com/bgauryy/FreeLlama/blob/main/README.md#
|
|
5
|
+
and Ollama-compatible passthrough. Use the [README CLI section](https://github.com/bgauryy/FreeLlama/blob/main/README.md#from-the-terminal) for first
|
|
6
6
|
use and this reference to look up commands and controls.
|
|
7
7
|
|
|
8
8
|
## Choose a command
|
|
@@ -78,8 +78,9 @@ npx @octocodeai/freellama serve \
|
|
|
78
78
|
--cpu-model nomic-embed-text:latest
|
|
79
79
|
```
|
|
80
80
|
|
|
81
|
-
FreeLlama rejects a CPU upstream with no assignments,
|
|
82
|
-
|
|
81
|
+
FreeLlama rejects a CPU upstream with no assignments, `--cpu-model` without `--cpu-upstream`, and a
|
|
82
|
+
CPU upstream that is the same Ollama endpoint as `--upstream`, including loopback aliases on one port.
|
|
83
|
+
It also sets `options.num_gpu=0` on CPU-assigned managed requests.
|
|
83
84
|
Read [CPU and GPU model routing](CPU_GPU_ROUTING.md) before deploying this layout.
|
|
84
85
|
|
|
85
86
|
After startup, require `contracts.placement_observation` and `placement_evidence_gate`, then inspect
|
|
@@ -145,7 +146,8 @@ FreeLlama first unloads idle unpinned runners, then dispatches anyway: the recei
|
|
|
145
146
|
load fits. On Linux, Windows, and other platforms, where available RAM already counts reclaimable
|
|
146
147
|
cache (and is the hard cap in a container), the same shortfall unloads idle runners and then holds the
|
|
147
148
|
new load with `held_insufficient_capacity` until capacity returns, refusing with 503 at its deadline.
|
|
148
|
-
OS memory pressure holds work
|
|
149
|
+
OS memory pressure holds all work where the OS reports it (macOS, and Linux when the kernel
|
|
150
|
+
exposes `/proc/pressure/memory`; Windows has no host collector yet). Active swapping and low RAM hold new loads only:
|
|
149
151
|
an already-resident runner adds no memory and stays admitted. CPU-load (one-minute load at or above
|
|
150
152
|
1.5 per logical CPU) and thermal-throttling holds apply to CPU-assigned work and to cold loads, which
|
|
151
153
|
decode weights on the host cores and add a new heat source; a bounded request on a runner already
|
|
@@ -153,12 +155,15 @@ resident on the GPU neither waits for the cores nor cools the package by waiting
|
|
|
153
155
|
lists the holds that applied to the request.
|
|
154
156
|
Both `serve` and `proxy` accept `--resource-telemetry-policy`:
|
|
155
157
|
|
|
156
|
-
- `require-memory` (default): wait or refuse local inference when available RAM is
|
|
157
|
-
raw requests with zero forecast bytes. Missing thermal data alone does not block
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
158
|
+
- `require-memory` (default on Linux and macOS): wait or refuse local inference when available RAM is
|
|
159
|
+
unknown, including raw requests with zero forecast bytes. Missing thermal data alone does not block
|
|
160
|
+
this policy.
|
|
161
|
+
- `require-all`: require all signals applicable to the collector. The built-in Linux collector reads
|
|
162
|
+
OS memory pressure from `/proc/pressure/memory` when the kernel provides it, but has no thermal
|
|
163
|
+
signal and marks it as unsupported; custom collectors must provide it. Unsupported signals are not
|
|
164
|
+
evidence of healthy hardware.
|
|
165
|
+
- `best-effort`: explicitly permit unknown telemetry unless an observed pressure hold is active. This
|
|
166
|
+
is the default on Windows, which has no built-in memory collector.
|
|
162
167
|
|
|
163
168
|
Remote upstreams bypass local host policy. Raw metadata and empty unload requests remain available
|
|
164
169
|
during a hold. Route receipts separate `queue_readiness` from `resource_readiness`, and include
|
|
@@ -320,8 +325,6 @@ The MCP server autostarts `serve` with its own environment and an arbitrary work
|
|
|
320
325
|
put a machine-wide policy in the config directory or set the variables in the MCP client's `env`.
|
|
321
326
|
`serve` prints each file it picked up.
|
|
322
327
|
|
|
323
|
-
Replace `MODEL` with the directory containing your completed quality-evaluation aggregate.
|
|
324
|
-
|
|
325
328
|
`policy from-eval` reads `pass_at_1`, not the throughput produced by `bench-all`. It refuses
|
|
326
329
|
expired aggregates and fewer than three trials unless `--allow-smoke` is explicit, and it skips
|
|
327
330
|
models that are not installed on `--ollama-endpoint` (default `FREELLAMA_OLLAMA_ENDPOINT`, then
|
package/docs/CPU_GPU_ROUTING.md
CHANGED
|
@@ -56,7 +56,7 @@ flowchart TB
|
|
|
56
56
|
CP --> CPU["CPU runner"]
|
|
57
57
|
```
|
|
58
58
|
|
|
59
|
-
FreeLlama does not
|
|
59
|
+
FreeLlama does not rewrite raw passthrough request bodies or route them by model. A direct Ollama client that
|
|
60
60
|
needs model-aware CPU placement must use the managed task endpoint instead.
|
|
61
61
|
|
|
62
62
|
## Start the backends
|
|
@@ -117,7 +117,7 @@ unsafe or ambiguous configurations:
|
|
|
117
117
|
- A CPU model assignment without `--cpu-upstream`
|
|
118
118
|
- Empty model names
|
|
119
119
|
- The same socket expressed through aliases such as `localhost:11434` and `127.0.0.1:11434`
|
|
120
|
-
-
|
|
120
|
+
- An upstream that points back to the FreeLlama listener, or an unspecified host such as `0.0.0.0`
|
|
121
121
|
|
|
122
122
|
A nonloopback FreeLlama listener requires `--allow-remote` and bearer-token authentication.
|
|
123
123
|
See the [production runbook](PRODUCTION.md) for remote ingress requirements.
|
|
@@ -216,7 +216,7 @@ curl --silent http://127.0.0.1:11436/api/ps | jq '.models[] | {name, size, size_
|
|
|
216
216
|
A fully CPU-loaded model reports `size_vram:0`; a fully GPU-loaded model reports positive VRAM
|
|
217
217
|
covering its resident `size`. Compare both values from the same `/api/ps` entry. Installed `/api/tags`
|
|
218
218
|
file bytes can differ from resident memory and cannot prove a mixed placement. Positive VRAM without
|
|
219
|
-
total resident size stays unknown. `observation.status` is `verified`, `mismatch`, or unavailable
|
|
219
|
+
total resident size stays unknown. `observation.status` is `verified`, `mismatch`, `not_resident`, or `unavailable`. Unknown, mixed, and mismatched
|
|
220
220
|
samples are reported but excluded from adaptive feedback.
|
|
221
221
|
|
|
222
222
|
## Understand concurrency
|
|
@@ -252,17 +252,18 @@ FreeLlama can overlap requests across the two servers even when each Ollama proc
|
|
|
252
252
|
`OLLAMA_NUM_PARALLEL=1`. Raising `OLLAMA_NUM_PARALLEL` affects parallel requests within one server
|
|
253
253
|
and multiplies K/V-cache memory; it is a separate tuning decision. `OLLAMA_MAX_QUEUE` likewise
|
|
254
254
|
bounds each Ollama process after FreeLlama admission. It does not replace either backend's weighted
|
|
255
|
-
FreeLlama budget, bounded waiter count, or queue-wait deadline. Raw passthrough uses a
|
|
256
|
-
one-stream default cap in `serve`; its permit is held until the streamed response ends or is dropped.
|
|
255
|
+
FreeLlama budget, bounded waiter count, or queue-wait deadline. Raw generation passthrough uses a
|
|
256
|
+
separate one-stream default cap in `serve`; its permit is held until the streamed response ends or is dropped.
|
|
257
257
|
|
|
258
258
|
The runtime file's `[task_costs]` table can tune individual task-kind weights without increasing
|
|
259
259
|
the entire pool's ceiling. Default vision weight four can serialize two vision requests under a
|
|
260
260
|
four-unit budget even when Ollama has two slots. A lower weight requires a qualified workload;
|
|
261
261
|
it does not add engine slots or relax memory checks. See
|
|
262
262
|
[monitoring controls](MONITORING.md#adaptive-concurrency).
|
|
263
|
-
|
|
264
|
-
lock and receive 503
|
|
265
|
-
|
|
263
|
+
Raw generation requests also require exclusive access to the primary backend's managed transition
|
|
264
|
+
lock and receive 503 if it stays busy past the raw queue wait. Metadata GET/HEAD, `/api/show`,
|
|
265
|
+
model-store calls (pull, push, copy, delete), and empty `keep_alive:0` unloads remain available.
|
|
266
|
+
This prevents raw inference from overlapping a managed generation on that backend.
|
|
266
267
|
|
|
267
268
|
## Understand automatic feedback
|
|
268
269
|
|
|
@@ -382,9 +383,8 @@ A configured but unreachable CPU backend causes model discovery to fail closed.
|
|
|
382
383
|
can also be unavailable until the secondary server returns. Raw passthrough to the primary server
|
|
383
384
|
continues to work because it does not depend on catalog discovery.
|
|
384
385
|
|
|
385
|
-
For the surrounding
|
|
386
|
-
|
|
387
|
-
[CLI reference](CLI.md).
|
|
386
|
+
For the surrounding Ollama settings, see [Production](PRODUCTION.md#configure-ollama-explicitly). For
|
|
387
|
+
all `serve` flags, see the [CLI reference](CLI.md).
|
|
388
388
|
|
|
389
389
|
## Upstream sources
|
|
390
390
|
|
package/docs/MODEL_SELECTION.md
CHANGED
|
@@ -32,8 +32,9 @@ flowchart TD
|
|
|
32
32
|
grounded research, image description, OCR, and summarization. Using one large multimodal model also
|
|
33
33
|
avoided contention between separate heavy text and vision runners.
|
|
34
34
|
|
|
35
|
-
|
|
36
|
-
and
|
|
35
|
+
In those evaluations it was strong at grounded lookup, long-context retrieval and strict instruction
|
|
36
|
+
following, needed `think:true` and a generous output budget for multi-step reasoning, and about a
|
|
37
|
+
third of its unsupervised code-review findings were false or context-limited.
|
|
37
38
|
These examples do not establish that a tag is installed or suitable on your host.
|
|
38
39
|
|
|
39
40
|
Dedicated OCR models answer best to the prompts their model card documents; send them with base64
|
package/docs/MONITORING.md
CHANGED
|
@@ -118,8 +118,9 @@ The ledger lives next to the feedback file in the platform data directory. Choos
|
|
|
118
118
|
`/metrics` exposes counters (`freellama_tasks_total`, `freellama_prompt_tokens_total`,
|
|
119
119
|
`freellama_output_tokens_total`, `freellama_task_seconds_total`, `freellama_queue_wait_seconds_total`,
|
|
120
120
|
`freellama_load_seconds_total`, `freellama_raw_requests_total`, `freellama_evictions_total`,
|
|
121
|
-
`freellama_circuit_open_total`, `freellama_concurrency_limit_changes_total`)
|
|
122
|
-
backend, task, and outcome,
|
|
121
|
+
`freellama_circuit_open_total`, `freellama_concurrency_limit_changes_total`); task counters are
|
|
122
|
+
labelled by model, backend, task, and outcome, raw requests by kind and outcome, and the rest by
|
|
123
|
+
backend. Gauges cover admission, queues, the raw proxy, host and GPU memory,
|
|
123
124
|
loaded models, and `freellama_ollama_num_parallel`. Point Prometheus, Grafana Agent, or any
|
|
124
125
|
OpenMetrics-compatible scraper at it.
|
|
125
126
|
|
|
@@ -132,7 +133,7 @@ FreeLlama waits for safe capacity within a deadline and bounds the retained back
|
|
|
132
133
|
| Managed queue full | `429` | `admission_queue_full` |
|
|
133
134
|
| Raw proxy over its cap after `raw_queue_wait_seconds` | `429` | |
|
|
134
135
|
| Managed wait exceeded `max_queue_wait_seconds` (or the task's `max_wait_seconds`) | `503` | `admission_timeout` |
|
|
135
|
-
| OS pressure, swapping, CPU, thermal, or required telemetry still holds when the waiting budget expires (low available RAM alone only warns; low RAM and swapping never hold an already-resident runner) | `503` | `resource_admission_unavailable` |
|
|
136
|
+
| OS pressure, swapping, CPU, thermal, or required telemetry still holds when the waiting budget expires (low available RAM alone only warns on macOS; low RAM and swapping never hold an already-resident runner, and CPU load and thermal never hold one on the GPU) | `503` | `resource_admission_unavailable` |
|
|
136
137
|
| Backend circuit open | `503` | `upstream_circuit_open` |
|
|
137
138
|
|
|
138
139
|
These capacity refusals carry a `Retry-After` header and `retry_after_seconds` in the body. For a queue it
|
|
@@ -267,9 +268,10 @@ out-of-memory shrink-and-retry only applies when the context was not requested e
|
|
|
267
268
|
|
|
268
269
|
With `context_mode = "ollama_default"` (the default), FreeLlama sizes the context as before but
|
|
269
270
|
omits `num_ctx` when Ollama's own default for the model is known, covers the request, and is at most
|
|
270
|
-
`auto_context_max` (32768). The default is
|
|
271
|
-
then `OLLAMA_CONTEXT_LENGTH`, then (
|
|
272
|
-
|
|
271
|
+
`auto_context_max` (32768). The default is the Modelfile's `num_ctx`, then the
|
|
272
|
+
runtime file's `ollama_default_context`, then `OLLAMA_CONTEXT_LENGTH`, then (primary backend
|
|
273
|
+
on a discrete GPU only) Ollama's VRAM tier, from total VRAM minus `OLLAMA_GPU_OVERHEAD` — 4096 below
|
|
274
|
+
23 GiB, 32768 below 47 GiB, 262144 from 47 GiB. Receipts show `context_sizing.num_ctx_sent` and the default used.
|
|
273
275
|
Memory estimates always use the context Ollama can allocate. Use `"explicit"` to always
|
|
274
276
|
send `num_ctx`.
|
|
275
277
|
|
package/docs/PRODUCTION.md
CHANGED
|
@@ -34,15 +34,14 @@ published tag:
|
|
|
34
34
|
powershell -ExecutionPolicy Bypass -File .\scripts\install.ps1 -Version VERSION
|
|
35
35
|
```
|
|
36
36
|
|
|
37
|
-
The repository has no CI or release workflows; they were removed (see git history).
|
|
38
|
-
|
|
39
|
-
|
|
37
|
+
The repository has no CI or release workflows; they were removed (see git history). Release from a
|
|
38
|
+
clean checkout: `yarn release:version <version>`, `yarn release:build` (builds each target's
|
|
39
|
+
executable and addon into `release-artifacts/<target>/`), `node scripts/assemble-release.mjs
|
|
40
|
+
release-artifacts release`, then `yarn release:verify:publish`. The publish verifier checks that
|
|
41
|
+
`FREELLAMA_RELEASE_TAG`, when set, matches the workspace version.
|
|
40
42
|
Live E2E and hardware qualification remain separate operator checks; verify those receipts before tagging.
|
|
41
43
|
|
|
42
|
-
|
|
43
|
-
environment, place each executable and matching addon under `release-artifacts/<target>/`, run
|
|
44
|
-
`node scripts/assemble-release.mjs release-artifacts release` and `yarn release:verify:publish`, then attach
|
|
45
|
-
`release/SHA256SUMS` and binaries to the release. Publish the eight `@octocodeai/freellama-native-*` packages
|
|
44
|
+
Attach `release/SHA256SUMS` and binaries to the release. Publish the eight `@octocodeai/freellama-native-*` packages
|
|
46
45
|
first, followed by the exact-version `@octocodeai/freellama` and `@octocodeai/freellama-mcp-server` packages. Configure
|
|
47
46
|
registry credentials in the release environment; never store them in the repository.
|
|
48
47
|
|
|
@@ -82,7 +81,7 @@ FreeLlama and Ollama have separate queues. FreeLlama caps retained managed waite
|
|
|
82
81
|
`--max-queued-tasks`/`--cpu-max-queued-tasks`, then waits for at most
|
|
83
82
|
`--max-queue-wait-seconds`. An admitted request can then enter Ollama's internal queue,
|
|
84
83
|
which is bounded by `OLLAMA_MAX_QUEUE`. Raw compatibility traffic bypasses weighted managed
|
|
85
|
-
admission and enters the primary Ollama queue through a separate one-stream default cap in `serve`.
|
|
84
|
+
admission and, for generation, enters the primary Ollama queue through a separate one-stream default cap in `serve`.
|
|
86
85
|
Set all limits from observed latency and overload behavior; changing one does not configure the
|
|
87
86
|
others.
|
|
88
87
|
|
|
@@ -91,7 +90,7 @@ The following flow shows where each production setting applies:
|
|
|
91
90
|
```mermaid
|
|
92
91
|
flowchart LR
|
|
93
92
|
C["Client"] --> FQ["FreeLlama admission<br/>weighted budget + wait deadline"]
|
|
94
|
-
FQ -->|"refused"| E503["503 server busy"]
|
|
93
|
+
FQ -->|"refused"| E503["429 or 503 server busy"]
|
|
95
94
|
FQ -->|"admitted"| OQ["Ollama queue<br/>OLLAMA_MAX_QUEUE"]
|
|
96
95
|
RAW["Raw /api/* or /v1/*"] --> RC["Raw stream cap<br/>default 1 in serve"]
|
|
97
96
|
RC -->|"refused"| E503
|
|
@@ -102,14 +101,24 @@ flowchart LR
|
|
|
102
101
|
|
|
103
102
|
These values are conservative starting points, not universal constants. NVIDIA, AMD, Apple, and
|
|
104
103
|
CPU-only hosts must pass hardware validation before promotion. See
|
|
105
|
-
[Ollama and FreeLlama optimization](dev/OLLAMA_SYSTEM_OPTIMIZATION.md) for the ownership boundary and
|
|
106
104
|
[Run models on CPU and GPU](CPU_GPU_ROUTING.md) for device-specific controls.
|
|
107
105
|
|
|
106
|
+
Ollama already picks the accelerator, fits the model to device memory, splits layers between CPU
|
|
107
|
+
and GPU, and schedules requests; FreeLlama decides which tasks reach it. These changes do not help:
|
|
108
|
+
|
|
109
|
+
- Forcing one model's layers between CPU and GPU: partial CPU offload usually lowers decode speed.
|
|
110
|
+
Run a separate CPU-only Ollama process for a second model instead.
|
|
111
|
+
- Running Ollama in Docker on macOS: Docker Desktop gives it no GPU acceleration there.
|
|
112
|
+
- Raising context globally: K/V memory grows with context; size `num_ctx` per request.
|
|
113
|
+
- Pinning several large models in one process without the memory arithmetic: weights, runtime
|
|
114
|
+
buffers and every active context must fit together.
|
|
115
|
+
- `q4_0` K/V cache as a default: it trades more quality than `q8_0` for the memory it saves.
|
|
116
|
+
|
|
108
117
|
Put the values in the service manager that owns each Ollama process: launchd on macOS, systemd or
|
|
109
118
|
the container definition on Linux, and the Windows service wrapper. Shell exports and
|
|
110
119
|
`launchctl setenv` are useful diagnostics but are login-session state, not a reboot-persistent
|
|
111
|
-
production configuration. After a real service restart, rerun `doctor` and `
|
|
112
|
-
require the same visible values before admitting work.
|
|
120
|
+
production configuration. After a real service restart, rerun `freellama doctor` and `freellama status`
|
|
121
|
+
and require the same visible values before admitting work.
|
|
113
122
|
|
|
114
123
|
## Create the security and state files
|
|
115
124
|
|
|
@@ -170,7 +179,8 @@ only model identifiers, scale, and sample count; they contain no prompts.
|
|
|
170
179
|
Keep adapter context settings distinct from Ollama server context. `FREELLAMA_AGENT_NUM_CTX`
|
|
171
180
|
budgets the adapter conversation; older tool observations are compacted inside that budget while
|
|
172
181
|
the system prompt and original question remain pinned by default. `OLLAMA_CONTEXT_LENGTH` controls
|
|
173
|
-
direct Ollama requests
|
|
182
|
+
direct Ollama requests; managed FreeLlama tasks size their own context and send `num_ctx` unless
|
|
183
|
+
Ollama's known default covers it.
|
|
174
184
|
|
|
175
185
|
## Verify lifecycle and placement
|
|
176
186
|
|
|
@@ -187,10 +197,10 @@ Require these health contracts before admitting traffic:
|
|
|
187
197
|
- `placement_observation: ollama_api_ps_after_execution`
|
|
188
198
|
- `placement_evidence_gate: configured_or_observed`
|
|
189
199
|
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
backend topology,
|
|
193
|
-
the Ollama process does not expose to FreeLlama.
|
|
200
|
+
With `FREELLAMA_AUTH_TOKEN_FILE` set, run `freellama doctor` (Ollama and CLI versions, visible Ollama
|
|
201
|
+
settings, machine) and read `/_freellama/v1/health` (FreeLlama version, `security`, `feedback.persistence`,
|
|
202
|
+
`backends`) and `freellama status` (backend topology, loaded models, admission). Also inspect the
|
|
203
|
+
service definition for any setting that the Ollama process does not expose to FreeLlama.
|
|
194
204
|
|
|
195
205
|
## Validate every hardware class
|
|
196
206
|
|
package/docs/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
FreeLlama helps agents offload bounded work to local models through Ollama. The calling agent owns
|
|
4
4
|
the task and verifies the answer; FreeLlama manages model selection and resources around execution.
|
|
5
|
-
Start with the [repository MCP setup](https://github.com/bgauryy/FreeLlama/blob/main/README.md#
|
|
5
|
+
Start with the [repository MCP setup](https://github.com/bgauryy/FreeLlama/blob/main/README.md#mcp-server) to run one managed task.
|
|
6
6
|
|
|
7
7
|
## Choose a guide
|
|
8
8
|
|
|
@@ -19,7 +19,7 @@ Start with the [repository MCP setup](https://github.com/bgauryy/FreeLlama/blob/
|
|
|
19
19
|
| Understand raw Ollama compatibility | [Ollama sidecar](ARCHITECTURE.md#ollama-sidecar) |
|
|
20
20
|
| Evaluate context offload and its cost | [Token economics](https://github.com/bgauryy/FreeLlama/blob/main/packages/mcp/README.md#token-economics) |
|
|
21
21
|
| Deploy the management layer | [Production runbook](PRODUCTION.md) |
|
|
22
|
-
| Run development and release checks | [Testing](TESTING.md), [release
|
|
22
|
+
| Run development and release checks | [Testing](TESTING.md), [release packages](TESTING.md#verify-release-packages) |
|
|
23
23
|
| Embed the routing core | [Rust core](https://github.com/bgauryy/FreeLlama/blob/main/packages/rust-core/README.md) |
|
|
24
24
|
| Use the read-only status page | `/_freellama/ui` on a running `serve` (see [Monitoring](MONITORING.md)) |
|
|
25
25
|
|
|
@@ -27,10 +27,6 @@ Start with the [repository MCP setup](https://github.com/bgauryy/FreeLlama/blob/
|
|
|
27
27
|
|
|
28
28
|
| Goal | Source |
|
|
29
29
|
|---|---|
|
|
30
|
-
| Review dated findings and project comparisons | [Findings report](dev/FINDINGS_AND_POSITIONING.md) |
|
|
31
|
-
| Measure inference, device activity, and thermal behavior | [Local performance testing](dev/LOCAL_PERFORMANCE_TESTING.md) |
|
|
32
|
-
| Separate management controls from inference tuning | [Ollama optimization boundary](dev/OLLAMA_SYSTEM_OPTIMIZATION.md) |
|
|
33
|
-
| Understand backend selection decisions | [Resource-routing ADR](dev/ADR_RESOURCE_AWARE_BACKEND_ROUTING.md) |
|
|
34
30
|
|
|
35
31
|
Historical measurements describe one model, machine, and workload. Inspect current state and
|
|
36
32
|
repeat the relevant checks before using them to select models or set resource limits.
|
package/docs/TESTING.md
CHANGED
|
@@ -24,8 +24,8 @@ yarn install
|
|
|
24
24
|
yarn build
|
|
25
25
|
```
|
|
26
26
|
|
|
27
|
-
The root build compiles the host Rust release binary, NAPI addon, and
|
|
28
|
-
|
|
27
|
+
The root build compiles the host Rust release binary, NAPI addon, and MCP JavaScript bundles
|
|
28
|
+
(server and research adapters). The native `.node` addon remains external to the JavaScript bundle because it is selected
|
|
29
29
|
by platform triple at runtime. Published packages resolve it from an OS/CPU/libc-specific optional
|
|
30
30
|
dependency rather than compiling Rust during installation.
|
|
31
31
|
|
|
@@ -40,7 +40,6 @@ Yarn. Use the `:js` or `:rust` variants to run one side only.
|
|
|
40
40
|
| All package unit tests (JS + Rust) | `yarn test` | Rust toolchain |
|
|
41
41
|
| JavaScript unit | `yarn test:js` | None |
|
|
42
42
|
| Rust contracts | `yarn test:rust` | Rust toolchain |
|
|
43
|
-
| Local-agent contracts | `yarn test:agents` | None after install; builds the MCP package |
|
|
44
43
|
| TypeScript | `yarn typecheck` | None after install |
|
|
45
44
|
| MCP integration | `yarn test:integration` | Ollama at `FREELLAMA_OLLAMA_ENDPOINT` (default `127.0.0.1:11434`) |
|
|
46
45
|
| MCP end to end | `yarn test:e2e` | Ollama, `freellama serve`, and required models |
|
|
@@ -137,11 +136,11 @@ Usage-ledger unit contracts cover ordered writes, bounded queue overflow, failed
|
|
|
137
136
|
validation, replay, and flush behavior. `monitoring_contract` also checks that a blocked ledger path
|
|
138
137
|
does not hold task completion. Persistence remains best effort, not a zero-loss promise.
|
|
139
138
|
|
|
140
|
-
MCP `serve.test.ts` checks startup permission failure, retry,
|
|
139
|
+
MCP `serve.test.ts` checks startup permission failure, retry, child-lifecycle isolation, and the
|
|
140
|
+
background start at launch. The
|
|
141
141
|
protocol integration contracts verify that spawn failure leaves the MCP connection usable and that
|
|
142
142
|
fixture-backed library requests fail on unexpected product errors. External catalog availability
|
|
143
|
-
is separate from these deterministic checks.
|
|
144
|
-
Windows static paths alongside traversal refusal.
|
|
143
|
+
is separate from these deterministic checks.
|
|
145
144
|
|
|
146
145
|
Launcher fixtures use `process.execPath` to avoid an extra PATH-resolved shell wrapper. After the
|
|
147
146
|
MCP fixture's real executable-permission refusal, a test-only transport launches that exact fixture
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@octocodeai/freellama-mcp-server",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.1",
|
|
4
4
|
"description": "MCP server exposing FreeLlama core as tools, via native NAPI bindings into packages/rust-core.",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -50,15 +50,16 @@
|
|
|
50
50
|
"zod": "^3.24.1"
|
|
51
51
|
},
|
|
52
52
|
"optionalDependencies": {
|
|
53
|
-
"@octocodeai/freellama-native-darwin-arm64": "0.6.
|
|
54
|
-
"@octocodeai/freellama-native-darwin-x64": "0.6.
|
|
55
|
-
"@octocodeai/freellama-native-linux-arm64-gnu": "0.6.
|
|
56
|
-
"@octocodeai/freellama-native-linux-arm64-musl": "0.6.
|
|
57
|
-
"@octocodeai/freellama-native-linux-x64-gnu": "0.6.
|
|
58
|
-
"@octocodeai/freellama-native-linux-x64-musl": "0.6.
|
|
59
|
-
"@octocodeai/freellama-native-win32-arm64-msvc": "0.6.
|
|
60
|
-
"@octocodeai/freellama-native-win32-x64-msvc": "0.6.
|
|
53
|
+
"@octocodeai/freellama-native-darwin-arm64": "0.6.1",
|
|
54
|
+
"@octocodeai/freellama-native-darwin-x64": "0.6.1",
|
|
55
|
+
"@octocodeai/freellama-native-linux-arm64-gnu": "0.6.1",
|
|
56
|
+
"@octocodeai/freellama-native-linux-arm64-musl": "0.6.1",
|
|
57
|
+
"@octocodeai/freellama-native-linux-x64-gnu": "0.6.1",
|
|
58
|
+
"@octocodeai/freellama-native-linux-x64-musl": "0.6.1",
|
|
59
|
+
"@octocodeai/freellama-native-win32-arm64-msvc": "0.6.1",
|
|
60
|
+
"@octocodeai/freellama-native-win32-x64-msvc": "0.6.1"
|
|
61
61
|
},
|
|
62
|
+
"author": "Guy Bary",
|
|
62
63
|
"license": "Apache-2.0 OR MIT",
|
|
63
64
|
"publishConfig": {
|
|
64
65
|
"access": "public",
|