catence 0.2.0-beta.10 → 0.2.0-beta.12

package/dist/elt/storage/progress-sidecar.js

@@ -0,0 +1,180 @@
+import { existsSync } from 'node:fs';
+import { mkdir, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { normalizeProgress, } from '../../contracts/progress.js';
+export const PROGRESS_SIDECAR_SUFFIX = '.progress.json';
+export const PROGRESS_SIDECAR_STALE_MS = 5 * 60 * 1000;
+export const PROGRESS_KEEPALIVE_MS = 30 * 1000;
+export const PROGRESS_PUBLISH_THROTTLE_MS = 2 * 1000;
+const TERMINAL_STAGES = new Set(['completed', 'failed', 'interrupted', 'timed_out']);
+export function progressSidecarPath(paths, runId) {
+    return path.join(paths.staging, 'garmin', `${runId}${PROGRESS_SIDECAR_SUFFIX}`);
+}
+/**
+ * Persist one progress heartbeat as a sidecar file next to the Garmin staging
+ * manifest. Reading this file needs no DuckDB lock, so `progress` stays live
+ * while a sync run holds the single-writer database lock.
+ */
+export async function writeProgressSidecar(paths, runId, state) {
+    const target = progressSidecarPath(paths, runId);
+    await mkdir(path.dirname(target), { recursive: true });
+    const temporary = `${target}.${Date.now()}.${Math.random().toString(36).slice(2, 10)}.tmp`;
+    await writeFile(temporary, JSON.stringify(state));
+    await rename(temporary, target);
+}
+export async function removeProgressSidecar(paths, runId) {
+    await rm(progressSidecarPath(paths, runId), { force: true });
+}
+/**
+ * Scan every persisted sidecar heartbeat for an athlete store. Corrupt or
+ * unreadable files are skipped so a torn write can never break progress reads.
+ */
+export async function readProgressSidecars(paths) {
+    const directory = path.join(paths.staging, 'garmin');
+    if (!existsSync(directory))
+        return [];
+    const files = (await readdir(directory)).filter((file) => file.endsWith(PROGRESS_SIDECAR_SUFFIX));
+    const states = [];
+    for (const file of files) {
+        try {
+            const state = normalizeProgress(JSON.parse(await readFile(path.join(directory, file), 'utf8')));
+            if (state)
+                states.push(state);
+        }
+        catch {
+            // Torn or partial sidecar write; it is not authoritative.
+        }
+    }
+    return states;
+}
+/** Active runs from sidecars: non-terminal stage and a fresh heartbeat. */
+export async function readRunningProgress(paths, staleAfterMs = PROGRESS_SIDECAR_STALE_MS) {
+    const now = Date.now();
+    const running = [];
+    for (const state of await readProgressSidecars(paths)) {
+        if (TERMINAL_STAGES.has(state.stage))
+            continue;
+        const ageMs = now - Date.parse(state.heartbeatAt);
+        if (!Number.isFinite(ageMs) || ageMs > staleAfterMs)
+            continue;
+        running.push(state);
+    }
+    return running;
+}
+/**
+ * Merge database-backed progress with sidecar heartbeats. Sidecars win for a
+ * given run (they are freshest); database rows fill runs that never emitted a
+ * sidecar (for example Intervals runs). Recent history is deduplicated across
+ * both sources and sorted by heartbeat recency.
+ */
+export function mergeProgress(dbRunning, dbRecent, sidecars) {
+    const running = new Map();
+    for (const state of sidecars)
+        running.set(state.runId, state);
+    for (const state of dbRunning)
+        if (!running.has(state.runId))
+            running.set(state.runId, state);
+    const recent = new Map();
+    for (const state of dbRecent)
+        recent.set(state.runId, state);
+    for (const state of sidecars)
+        recent.set(state.runId, state);
+    return {
+        running: [...running.values()],
+        recent: [...recent.values()]
+            .sort((a, b) => Date.parse(b.heartbeatAt) - Date.parse(a.heartbeatAt))
+            .slice(0, 10),
+    };
+}
+function baselineState(runId, provider, stage) {
+    return {
+        runId,
+        provider,
+        stage,
+        currentStep: null,
+        completedUnits: 0,
+        totalUnits: null,
+        percentComplete: 0,
+        elapsedSeconds: 0,
+        estimatedRemainingSeconds: null,
+        heartbeatAt: new Date().toISOString(),
+    };
+}
+/**
+ * Drives progress heartbeats for the whole lifetime of a sync run - including
+ * phases that emit no worker output (the Intervals extractor and the Garmin
+ * JSONL import). `publish` throttles writes to the database and sidecar; a
+ * 30-second keep-alive interval refreshes the last known state so a run never
+ * looks stale while it is still active, even during the lock-holding import.
+ */
+export class ProgressPump {
+    runId;
+    provider;
+    options;
+    lastProgress = null;
+    lastHeartbeatAt = 0;
+    interval = null;
+    stageOverride = null;
+    constructor(runId, provider, options) {
+        this.runId = runId;
+        this.provider = provider;
+        this.options = options;
+    }
+    start() {
+        if (this.interval)
+            return;
+        this.interval = setInterval(() => {
+            this.publish(this.lastProgress
+                ? { ...this.lastProgress, heartbeatAt: new Date().toISOString() }
+                : baselineState(this.runId, this.provider, this.stageOverride ?? 'starting'));
+        }, PROGRESS_KEEPALIVE_MS);
+        this.interval.unref();
+    }
+    publish(state) {
+        const now = Date.now();
+        if (this.lastHeartbeatAt !== 0 && now - this.lastHeartbeatAt < PROGRESS_PUBLISH_THROTTLE_MS) {
+            this.lastProgress = state;
+            return;
+        }
+        this.lastHeartbeatAt = now;
+        this.lastProgress = state;
+        const { database, paths, log } = this.options;
+        void database.heartbeatRun(this.runId, state).catch((error) => {
+            log.debug('Progress heartbeat failed', { runId: this.runId, error: error instanceof Error ? error.message : String(error) });
+        });
+        void writeProgressSidecar(paths, this.runId, state).catch((error) => {
+            log.debug('Progress sidecar write failed', { runId: this.runId, error: error instanceof Error ? error.message : String(error) });
+        });
+    }
+    /** Force subsequent keep-alive heartbeats to report the given stage. */
+    setStage(stage) {
+        this.stageOverride = stage;
+        if (this.lastProgress) {
+            this.lastProgress = { ...this.lastProgress, stage };
+            this.publish(this.lastProgress);
+        }
+    }
+    /** Transition the run to the JSONL import stage; keep-alives persist it. */
+    setStageImporting() {
+        this.setStage('importing');
+    }
+    /** Write a terminal heartbeat immediately, bypassing the publish throttle. */
+    publishFinal(state) {
+        this.lastHeartbeatAt = Date.now();
+        this.lastProgress = state;
+        const { database, paths, log } = this.options;
+        void database.heartbeatRun(this.runId, state).catch((error) => {
+            log.debug('Progress heartbeat failed', { runId: this.runId, error: error instanceof Error ? error.message : String(error) });
+        });
+        void writeProgressSidecar(paths, this.runId, state).catch((error) => {
+            log.debug('Progress sidecar write failed', { runId: this.runId, error: error instanceof Error ? error.message : String(error) });
+        });
+    }
+    stop() {
+        if (this.interval) {
+            clearInterval(this.interval);
+            this.interval = null;
+        }
+    }
+}
+//# sourceMappingURL=progress-sidecar.js.map

package/docs/architecture.md

@@ -87,7 +87,17 @@ The query layer has four shared pieces:
 
 ### 4. Serving and retrieval
 
-The server exposes `catence_status`, `describe_data`, `describe_dataset`, `read_series`, `aggregate_data`, `analyze_series`, `fit_series_model`, `query_read_only_data`, `find_activities`, `get_ftp_history`, `get_vo2max_history`, `power_curve_trend`, `power_coverage_report`, `latest_cycling_activities`, `cycling_progress_report`, `review_daily_recovery_load`, `review_weekly_training`, `review_activity_deep_dive`, `search_context`, `hydrate_strava_activity`, `hydrate_recent_strava_activities`, and `hydrate_strava_segment_history`, plus resources for status, catalog, a single activity, and a date-range summary.
+The server exposes these tools, grouped by purpose:
+
+- Catalog and status: `list_athletes`, `catence_status`, `catence_sync_progress`.
+- Review workflows: `review_daily_recovery_load`, `review_weekly_training`, `review_activity_deep_dive`, plus the matching `daily_recovery_load_review`, `weekly_training_review`, and `activity_deep_dive` prompts.
+- General data and analytics: `describe_data`, `describe_dataset`, `read_series`, `aggregate_data`, `analyze_series`, `fit_series_model`, `query_read_only_data`, `search_context`.
+- Wellness: `wellness_correlate`, `wellness_baselines`, `wellness_anomalies`, `wellness_coverage`.
+- Fitness and progress: `get_ftp_history`, `get_vo2max_history`, `power_curve_trend`, `power_coverage_report`, `latest_cycling_activities`, `cycling_progress_report`, `get_swim_laps`, `swim_progress_report`.
+- Activity detail and segments: `find_activities`, `get_activity_segments`.
+- Strava hydration (the only writers): `hydrate_strava_activity`, `hydrate_recent_strava_activities`, `hydrate_strava_segment_history`.
+
+Resources are mode-dependent. In catalog mode the server registers only `athletes` (`catence://athletes`), the machine-readable roster of configured athlete IDs and labels; in a single-store (non-catalog) deployment it registers resources for status, catalog, a single activity template (`catence://activity/{activityId}`), and a date-range summary template (`catence://summary/{startDate}/{endDate}`).
 
 `catence serve` is a loopback-only-by-default Streamable HTTP adapter. It creates a stateless MCP server and transport for each `/mcp` request, so the HTTP and stdio surfaces use the same tool implementation. `GET /health` is a versioned client handshake with runtime version, protocol version, and capabilities; distributed Console builds reject an incompatible protocol before starting a chat. The same listener serves a small read-only `/api/v1/dashboard` snapshot for the local Console: canonical daily-health rows, weekly canonical activity aggregates, recent canonical activities, status, coverage, and explicit missing-data caveats.
 

package/docs/configuration.md

@@ -0,0 +1,221 @@
+# Catence configuration reference
+
+This is the complete schema for `config.json` in the data home
+(`~/.catence/config.json` locally, `/data/config.json` in Docker). It is
+validated by the runtime before every run: unknown fields or invalid values
+produce an `Invalid Catence config at <path>: …` error. A missing or empty
+file is treated as `{}` (no rate limits, no budget, no Console profiles).
+
+The file is shared between the runtime (MCP rate limits, Strava budget) and
+the Console (model profiles and limits). It deliberately contains **no
+credential values** — only the *names* of environment variables that hold the
+values. See [`config.example.json`](../config.example.json) for a complete
+worked example and [`llm-providers.md`](llm-providers.md) for the provider
+profile how-to.
+
+## Top-level sections
+
+```jsonc
+{
+  "mcp":        { "rateLimits": { … } },   // optional
+  "providers":  { "strava": { "budget": { … } } }, // optional
+  "console":    { "defaultProfile": …, "limits": { … }, "profiles": { … } } // optional
+}
+```
+
+Every section is optional and strict: the schema rejects unknown top-level
+keys and unknown keys inside each section.
+
+## `mcp.rateLimits`
+
+Limits the MCP server's sliding-window limiter. Each entry is
+`{ "requests": <positive int>, "windowSeconds": <positive int> }` or `null`.
+
+```jsonc
+{
+  "mcp": {
+    "rateLimits": {
+      "server": { "requests": 240, "windowSeconds": 60 },
+      "tools": {
+        "*": { "requests": 60, "windowSeconds": 60 },
+        "hydrate_strava_activity": { "requests": 6, "windowSeconds": 60 },
+        "hydrate_recent_strava_activities": { "requests": 1, "windowSeconds": 60 },
+        "hydrate_strava_segment_history": { "requests": 4, "windowSeconds": 60 }
+      },
+      "resources": {
+        "*": { "requests": 120, "windowSeconds": 60 }
+      }
+    }
+  }
+}
+```
+
+- `tools` keys are tool names; `resources` keys are resource names; `*` is the
+  catch-all default for that kind.
+- A specific entry overrides `*`, which overrides `server`.
+- **`null` disables the limit** for that specific tool/resource name (or for
+  the whole server when set on `server`). This is the escape hatch for
+  long-running hydration tools.
+- Limits are enforced **per athlete**: each athlete gets its own window
+  (keyed `athleteId:tool:<name>` / `athleteId:resource:<name>`).
+
+## `providers.strava.budget`
+
+Throttles Strava API usage during sync and hydration.
+
+```jsonc
+{
+  "providers": {
+    "strava": {
+      "budget": {
+        "maxConcurrentRequests": 1,
+        "readRequestsPer15Minutes": 80,
+        "readRequestsPerDay": 800
+      }
+    }
+  }
+}
+```
+
+- `readRequestsPer15Minutes` and `readRequestsPerDay` are enforced: a sync or
+  hydration throws a retryable `rate_limited` / `StravaRateLimitError` when
+  the rolling usage reaches either threshold, and when Strava itself reports a
+  `blocked_until` timestamp.
+- `maxConcurrentRequests` is validated and documented but **not yet enforced**
+  by the current sync/hydration worker (hydration already runs serially); it
+  is reserved for a future concurrent worker. Set each value to `null` to
+  disable that particular check.
+
+## `console`
+
+The Console model configuration. `defaultProfile` must name one of
+`profiles`.
+
+```jsonc
+{
+  "console": {
+    "defaultProfile": "openai",
+    "limits": { "toolRounds": 8, "toolResultCharacters": 24000 },
+    "profiles": { … }
+  }
+}
+```
+
+### `console.limits`
+
+| Field | Default | Range |
+| --- | --- | --- |
+| `toolRounds` | 8 | 1–32 — max model tool-calling iterations per chat turn |
+| `toolResultCharacters` | 24000 | 1,000–250,000 — max characters of a tool result passed to the model (truncated past the limit) |
+
+### `console.profiles`
+
+Each profile holds one set of credentials and one or more model deployments:
+
+```jsonc
+"profiles": {
+  "openai": {
+    "label": "OpenAI",
+    "model": "openai/gpt-5-mini",
+    "apiKeyEnv": "OPENAI_API_KEY"
+  }
+}
+```
+
+Allowed profile fields (strict):
+
+| Field | Required | Meaning |
+| --- | --- | --- |
+| `label` | — | Display name in the Model dropdown |
+| `model` | exactly one of `model`/`models` | Single LiteLLM model name, e.g. `openai/gpt-5-mini` |
+| `models` | exactly one of `model`/`models` | Map of deployment id → per-model object for multiple deployments |
+| `defaultModel` | with `models` | Which deployment id is preselected; must name one of `models` (defaults to the first) |
+| `defaultReasoningEffort` | — | One of `minimal`, `low`, `medium`, `high`, `xhigh` |
+| `apiKeyEnv` | — | Name of the env var holding the API key |
+| `apiBaseEnv` | — | Name of the env var holding the base URL (OpenAI-compatible endpoints) |
+| `apiVersionEnv` | — | Name of the env var holding an API version, for endpoints that require one |
+
+Every `*Env` value must be a valid uppercase environment-variable name
+(`/^[A-Z][A-Z0-9_]*$/`); a raw secret there is rejected. Credentials are read
+from the environment only, never persisted.
+
+Per-model objects (under `models`):
+
+| Field | Meaning |
+| --- | --- |
+| `label` | Display name |
+| `model` | LiteLLM model name, e.g. `openai/deepseek-v4-flash-free` |
+| `reasoningEffort` | Fixed reasoning-effort level for every request |
+| `variants` | Map of dropdown label → exact value sent to the provider (non-OpenAI schemes); `{}` disables reasoning effort for the model |
+
+Validation rules:
+
+- A profile must define **exactly one** of `model` or `models`.
+- `defaultModel` requires `models` and must name one of its keys.
+- `defaultReasoningEffort` must be one of the OpenAI-standard levels.
+- A model cannot combine `variants: {}` with a fixed `reasoningEffort`.
+- `console.profiles` must be non-empty; `models` must be non-empty.
+
+Precedence for reasoning effort: per-chat dropdown selection > model
+`reasoningEffort` > profile `defaultReasoningEffort` > provider default. See
+[`llm-providers.md`](llm-providers.md#reasoning-effort) for the full behavior.
+
+### The Console wizard
+
+On the first authenticated chat, the Console wizard (`write_provider_setup`)
+offers presets for `openai-compatible`, `openai`, or `anthropic`. It writes a
+single-profile `console` section and **replaces the whole `console` section**,
+so re-running it overwrites any hand-edited profiles. The discovery scripts
+(`discover:opencode-go`, `discover:opencode-zen`) instead **merge** their
+profiles and preserve existing ones, limits, and `defaultProfile` unless
+`--set-default` is passed.
+
+## Environment variables
+
+### Catalog home and runtime
+
+| Variable | Purpose |
+| --- | --- |
+| `CATENCE_HOME` | Catalog home (default `~/.catence`); equivalent to `--home` |
+| `CATENCE_HTTP_HOST` / `CATENCE_HTTP_PORT` | Defaults for `catence serve --host/--port` (host `127.0.0.1`, port `8787`) |
+
+Provider credentials are normally stored per athlete with
+`catence-data secret set` (see [`local-mcp.md`](deployment/local-mcp.md#3-store-provider-credentials-stdin-only-never-shell-history)); those secrets are injected
+into the provider worker environment, and inherited `GARMIN_*`/`INTERVALS_*`/
+`STRAVA_*` variables are stripped first so an athlete without a provider cannot
+inherit its credentials.
+
+### Console auth and serving
+
+| Variable | Purpose |
+| --- | --- |
+| `CATENCE_CONSOLE_USERNAME` | Console login username (required) |
+| `CATENCE_CONSOLE_PASSWORD_HASH` | bcrypt hash for the login password (required; `catence-console auth hash-password`) |
+| `CHAINLIT_AUTH_SECRET` | JWT signing secret for the Console session (required) |
+| `CATENCE_HOME` | Data home served by the Console (default `~/.catence`) |
+| `CATENCE_MCP_URL` | MCP endpoint for the Console (default `http://127.0.0.1:8787/mcp`; when unset the Console auto-starts a runtime) |
+| `CATENCE_CONSOLE_HOST` | Default for `--ui-host` (`127.0.0.1`) |
+
+### Model providers
+
+Each env var name is referenced by a profile's `apiKeyEnv` / `apiBaseEnv` /
+`apiVersionEnv`; see [`llm-providers.md`](llm-providers.md) for the full
+provider guide. Common ones from [`config.example.json`](../config.example.json):
+
+| Variable | Profile |
+| --- | --- |
+| `OPENAI_API_KEY`, `OPENAI_API_BASE`, `OPENAI_API_VERSION` | `openai`, `openai-compatible`, Azure, Opencode proxy |
+| `ANTHROPIC_API_KEY` | `anthropic` |
+| `OPENCODE_GO_API_KEY`, `OPENCODE_GO_API_BASE`, `OPENCODE_GO_MESSAGES_API_BASE` | `opencode-go`, `opencode-go-messages` |
+| `OPENCODE_API_KEY`, `OPENCODE_ZEN_API_BASE`, `OPENCODE_ZEN_RESPONSES_API_BASE` | `opencode-zen`, `opencode-zen-responses` |
+
+### Docker (`.env`, generated by `deploy-console.sh`)
+
+| Variable | Purpose |
+| --- | --- |
+| `CATENCE_NPM_VERSION` | npm package version to install in the image |
+| `CATENCE_CONSOLE_VERSION` | PyPI `catence-console` version (PEP 440) |
+| `CATENCE_CONSOLE_PORT` | Host port for the Console (default 8000) |
+| `CATENCE_MCP_BIND` | MCP bind: `127.0.0.1` loopback (default) or `0.0.0.0` for network/Tailscale exposure; also drives the compose port mapping |
+
+See [`docker.md`](deployment/docker.md) for the Docker workflow.