catence 0.2.0 → 0.2.1-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/config.example.json +1 -0
- package/dist/contracts/release.d.ts +1 -1
- package/dist/contracts/release.js +1 -1
- package/dist/contracts/runtime.d.ts +4 -0
- package/dist/core/query/catalog.d.ts +1 -1
- package/dist/core/query/catalog.js +20 -10
- package/dist/core/query/repository.d.ts +1 -1
- package/dist/core/query/repository.js +31 -2
- package/dist/core/query/sql-guard.js +37 -5
- package/dist/core/runtime/athlete-file.d.ts +49 -0
- package/dist/core/runtime/athlete-file.js +177 -0
- package/dist/core/runtime/configuration.js +2 -0
- package/dist/elt/application/management.d.ts +7 -0
- package/dist/elt/application/management.js +94 -5
- package/dist/elt/normalization/normalizers.js +122 -0
- package/dist/elt/storage/migrations.js +45 -1
- package/dist/elt/storage/progress-sidecar.d.ts +9 -0
- package/dist/elt/storage/progress-sidecar.js +31 -16
- package/dist/interfaces/cli/commands.js +5 -1
- package/dist/interfaces/http/server.js +141 -3
- package/dist/interfaces/mcp/server.js +106 -16
- package/dist/runtime/index.d.ts +3 -1
- package/dist/runtime/index.js +2 -1
- package/docs/architecture.md +1 -1
- package/docs/configuration.md +49 -0
- package/docs/deployment/docker.md +87 -8
- package/docs/deployment/local-console.md +124 -21
- package/docs/deployment/local-mcp.md +12 -5
- package/docs/development.md +5 -3
- package/docs/llm-providers.md +4 -4
- package/package.json +7 -5
- package/pyproject.toml +1 -1
- package/python/catence/providers/garmin/cli.py +31 -3
- package/python/catence/providers/garmin/progress.py +71 -35
- package/python/catence/providers/garmin/staging.py +11 -4
- package/python/catence/providers/garmin/worker.py +420 -206
- package/uv.lock +3 -3
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
Catence Console is a password-protected local web chat that starts a matching
|
|
4
4
|
Catence runtime on loopback. It uses Chainlit password login and fails closed
|
|
5
|
-
unless
|
|
5
|
+
unless `CHAINLIT_AUTH_SECRET` and at least one login source — a stored Console
|
|
6
|
+
account or the environment account pair — are configured.
|
|
6
7
|
|
|
7
8
|
This guide covers the **local** path: the npm package plus the Python Console
|
|
8
9
|
on your machine. The Docker path (one container built from the public
|
|
@@ -27,6 +28,7 @@ model.
|
|
|
27
28
|
| Provider credentials | per-athlete secret file, mode 0600 | `catence-data secret set` |
|
|
28
29
|
| API keys | process environment only | shell exports |
|
|
29
30
|
| Chat history + preferences | data home | `console/chat-history.sqlite3` |
|
|
31
|
+
| Console logins + athlete grants | data home (mode 0600, bcrypt hashes) | `console/accounts.json` |
|
|
30
32
|
|
|
31
33
|
Credentials never land in `config.json`; profiles reference environment
|
|
32
34
|
variable *names* (`apiKeyEnv`, `apiBaseEnv`, `apiVersionEnv`). See
|
|
@@ -95,15 +97,28 @@ To add OpenCode Go models, run the
|
|
|
95
97
|
|
|
96
98
|
```sh
|
|
97
99
|
export OPENAI_API_KEY='…' # or ANTHROPIC_API_KEY / OPENCODE_GO_API_KEY …
|
|
98
|
-
export CATENCE_CONSOLE_USERNAME='coach'
|
|
99
|
-
export CATENCE_CONSOLE_PASSWORD_HASH="$(catence-console auth hash-password)"
|
|
100
100
|
export CHAINLIT_AUTH_SECRET="$(openssl rand -hex 32)"
|
|
101
101
|
```
|
|
102
102
|
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
103
|
+
`CHAINLIT_AUTH_SECRET` is always required: it signs Chainlit sessions. On top
|
|
104
|
+
of it, the Console needs at least one login source:
|
|
105
|
+
|
|
106
|
+
- **Stored accounts** (the multi-user path). Each account carries a role and
|
|
107
|
+
an athlete grant list and lives in `<home>/console/accounts.json`. Create
|
|
108
|
+
the first one with the users CLI, which prompts for the password twice:
|
|
109
|
+
|
|
110
|
+
```sh
|
|
111
|
+
catence-console users add coach --admin
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
- **The environment break-glass account**: set both
|
|
115
|
+
`CATENCE_CONSOLE_USERNAME` and `CATENCE_CONSOLE_PASSWORD_HASH` (the bcrypt
|
|
116
|
+
hash printed by `catence-console auth hash-password`). The pair is never
|
|
117
|
+
persisted and always acts as an admin — the way back in when the accounts
|
|
118
|
+
file is lost.
|
|
119
|
+
|
|
120
|
+
Roles, grants, and the rest of the CLI are described under
|
|
121
|
+
[Console accounts](#console-accounts).
|
|
107
122
|
|
|
108
123
|
## 7. Start the Console
|
|
109
124
|
|
|
@@ -121,7 +136,7 @@ catence-console doctor
|
|
|
121
136
|
|
|
122
137
|
### Manage models in the Console
|
|
123
138
|
|
|
124
|
-
The **Models**
|
|
139
|
+
The **Settings → Models** tab manages the model list without editing
|
|
125
140
|
`config.json` by hand:
|
|
126
141
|
|
|
127
142
|
- Enable/disable toggles hide models from the chat's Model dropdown. Disabled
|
|
@@ -137,11 +152,87 @@ The **Models** page (header button) manages the model list without editing
|
|
|
137
152
|
strict parser used at startup before it replaces the file, and secrets stay
|
|
138
153
|
forbidden (only environment-variable names are ever stored).
|
|
139
154
|
|
|
140
|
-
|
|
155
|
+
**Settings → Status** has a **Sync data** button that starts the same detached
|
|
141
156
|
sync as `catence-data sync --provider all` through the authenticated Console
|
|
142
157
|
origin, shows live progress while the run is active, and displays the last
|
|
143
|
-
completed sync afterwards
|
|
144
|
-
|
|
158
|
+
completed sync afterwards; the Dashboard keeps a read-only “Syncing…” pill and
|
|
159
|
+
links there when a write lock makes its data unavailable. Each manual sync also
|
|
160
|
+
refreshes OpenCode Go model profiles first; a discovery failure never blocks the
|
|
161
|
+
data sync.
|
|
162
|
+
|
|
163
|
+
### Add athletes in the Console
|
|
164
|
+
|
|
165
|
+
Admins can add a catalog athlete from **Settings → Athletes** (athlete id,
|
|
166
|
+
display name, and optionally “set as default”). The form calls the runtime's
|
|
167
|
+
`POST /api/v1/athletes` through the Console, which rejects members with
|
|
168
|
+
`403 admin_required` and otherwise performs the same catalog write as
|
|
169
|
+
`catence-data athlete add --id <id> --label <label>`. The id must match
|
|
170
|
+
`^[a-z][a-z0-9-]{0,62}$` and must not already exist (`409 athlete_exists`);
|
|
171
|
+
single-store runtimes without a catalog refuse with `400 catalog_required`.
|
|
172
|
+
The roster refreshes immediately, so a new athlete can be granted to members
|
|
173
|
+
and selected in chat without restarting the Console.
|
|
174
|
+
|
|
175
|
+
## Console accounts
|
|
176
|
+
|
|
177
|
+
Console accounts are Catence's multi-user login model, enforced entirely by
|
|
178
|
+
the Console: sign-in, the athlete roster, dashboard and athlete-file requests,
|
|
179
|
+
and the athlete scoping of every chat turn are all checked server-side against
|
|
180
|
+
the account's role and athlete grants. The MCP server itself is untouched — it
|
|
181
|
+
has no authentication and trusts the `athleteId` each call names (see
|
|
182
|
+
[local-mcp.md](local-mcp.md#streamable-http-server)); accounts only gate what
|
|
183
|
+
goes through the Console.
|
|
184
|
+
|
|
185
|
+
| Role | Athlete access |
|
|
186
|
+
| --- | --- |
|
|
187
|
+
| `admin` | Every athlete in the catalog (`athletes: "all"`) |
|
|
188
|
+
| `member` | Only the explicitly granted athletes; zero grants allowed |
|
|
189
|
+
|
|
190
|
+
Accounts live in `<home>/console/accounts.json` (mode 0600, bcrypt hashes
|
|
191
|
+
only), where `<home>` is `$CATENCE_HOME` or `~/.catence`. The Console re-reads
|
|
192
|
+
the file on every login, so new accounts, password resets, and grant changes
|
|
193
|
+
apply without a restart.
|
|
194
|
+
|
|
195
|
+
### Manage accounts with the users CLI
|
|
196
|
+
|
|
197
|
+
```sh
|
|
198
|
+
catence-console users add coach --admin # admin: all athletes
|
|
199
|
+
catence-console users add martina --athlete martina # member with one grant
|
|
200
|
+
catence-console users add sam # member, zero grants
|
|
201
|
+
catence-console users list
|
|
202
|
+
catence-console users passwd martina
|
|
203
|
+
catence-console users set-role sam member
|
|
204
|
+
catence-console users grant sam martina
|
|
205
|
+
catence-console users revoke sam martina
|
|
206
|
+
catence-console users remove sam
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
- `add <username> (--admin | --athlete <id> [--athlete <id> …])` — members
|
|
210
|
+
default to zero grants; `--admin` cannot be combined with `--athlete` and
|
|
211
|
+
always stores access to every athlete.
|
|
212
|
+
- `add` and `passwd` prompt for the password twice (never echoed). For
|
|
213
|
+
automation, `--password-env VAR` reads the password from environment
|
|
214
|
+
variable `VAR` instead, so it never appears in the command line or shell
|
|
215
|
+
history.
|
|
216
|
+
- `grant`/`revoke` change a member's athlete list; revoking the last grant
|
|
217
|
+
leaves a zero-grant member. `set-role` promotes or demotes: promoting to
|
|
218
|
+
`admin` coerces the grants to `"all"`, demoting to `member` keeps them.
|
|
219
|
+
- `add`/`grant`/`revoke` validate athlete ids against `catalog.json` when it
|
|
220
|
+
exists, and `--home <dir>` points any subcommand at a non-default catalog.
|
|
221
|
+
|
|
222
|
+
A member with zero grants can sign in and browse the Console, but there is no
|
|
223
|
+
athlete to scope a chat to: the Athlete selector is disabled with *"You don't
|
|
224
|
+
have access to any athlete yet. Ask an administrator to grant you access."*
|
|
225
|
+
and every chat turn is answered with the same notice. A member that requests
|
|
226
|
+
an athlete outside their grants receives `403 athlete_forbidden`; the Console
|
|
227
|
+
never forwards the request to the runtime.
|
|
228
|
+
|
|
229
|
+
### The environment break-glass account
|
|
230
|
+
|
|
231
|
+
`CATENCE_CONSOLE_USERNAME` and `CATENCE_CONSOLE_PASSWORD_HASH` must be set
|
|
232
|
+
together. When they are, the pair always acts as an `admin` and is never
|
|
233
|
+
written to `accounts.json`: unsetting the two variables revokes it. Use it as
|
|
234
|
+
the recovery path when the accounts file is lost or a lockout leaves nobody
|
|
235
|
+
who can grant access — not as the everyday login.
|
|
145
236
|
|
|
146
237
|
## Console serve options
|
|
147
238
|
|
|
@@ -167,12 +258,18 @@ chat starts.
|
|
|
167
258
|
|
|
168
259
|
## How multi-athlete works in the Console
|
|
169
260
|
|
|
170
|
-
The Console shares the same per-athlete stores as MCP
|
|
171
|
-
|
|
261
|
+
The Console shares the same per-athlete stores as MCP. Access is layered:
|
|
262
|
+
per-account grants at the login, then server-owned scoping per chat.
|
|
172
263
|
|
|
264
|
+
- **Per-account grants.** Each stored account is an `admin` (every athlete) or
|
|
265
|
+
a `member` with an explicit grant list (see
|
|
266
|
+
[Console accounts](#console-accounts)). The **Athlete** selector and the
|
|
267
|
+
proxied roster are narrowed to the account's grants; requests that name an
|
|
268
|
+
ungranted athlete are refused with `403 athlete_forbidden` before they reach
|
|
269
|
+
the runtime, and a zero-grant member sees no athletes at all.
|
|
173
270
|
- The settings panel has an **Athlete** selector built from
|
|
174
271
|
`GET /api/v1/athletes` on the runtime. The default is the catalog's
|
|
175
|
-
`defaultAthleteId
|
|
272
|
+
`defaultAthleteId` (for a member, the first granted athlete).
|
|
176
273
|
- On every personal-data tool call, the Console **forces** the selected
|
|
177
274
|
athleteId onto the arguments — the model cannot name or switch athletes, and
|
|
178
275
|
a chat's system message states: *"This Console chat is scoped to athleteId X.
|
|
@@ -183,10 +280,12 @@ The Console shares the same per-athlete stores as MCP, but scoping is
|
|
|
183
280
|
announces *"This chat is scoped to athlete **<id>**."*
|
|
184
281
|
- `list_athletes` is exempt from forcing so the roster can be rendered.
|
|
185
282
|
|
|
186
|
-
|
|
187
|
-
middleware proxies
|
|
188
|
-
|
|
189
|
-
|
|
283
|
+
Console pages and chats reach the runtime through the authenticated Console
|
|
284
|
+
origin: Chainlit middleware proxies the Console's API surface — `GET
|
|
285
|
+
/api/v1/dashboard`, `GET`/`POST /api/v1/athletes` (POST is admin-only), `GET
|
|
286
|
+
/api/v1/threads/<id>/generation`, and the models, accounts, tool-server, and
|
|
287
|
+
sync routes — to the runtime only when the Console's JWT cookie is valid
|
|
288
|
+
(otherwise 401), so raw port 8787 does not need to be exposed.
|
|
190
289
|
|
|
191
290
|
## Model discovery (OpenCode Go)
|
|
192
291
|
|
|
@@ -205,8 +304,8 @@ disappeared from the live catalog.
|
|
|
205
304
|
npm run discover:opencode-go -- --write ~/.catence/config.json
|
|
206
305
|
```
|
|
207
306
|
|
|
208
|
-
Inside
|
|
209
|
-
|
|
307
|
+
Inside **Settings → Models**, **Discover OpenCode Go models** runs the same
|
|
308
|
+
merge for the current home without touching athlete data (runtime route
|
|
210
309
|
`POST /api/v1/models/discover`, proxied behind the Console login). Models can
|
|
211
310
|
also be edited in place there — the update action rewrites the `console`
|
|
212
311
|
section atomically and never stores secret values.
|
|
@@ -260,4 +359,8 @@ uv tool install --upgrade catence-console
|
|
|
260
359
|
environment variables; set them in the shell and restart the Console.
|
|
261
360
|
- **Runtime not reachable** — `catence-console doctor --home "$HOME/.catence"
|
|
262
361
|
--mcp-url http://127.0.0.1:8787/mcp` checks the handshake; confirm the
|
|
263
|
-
runtime is running on that port and the protocol versions agree.
|
|
362
|
+
runtime is running on that port and the protocol versions agree.
|
|
363
|
+
- **Locked out of the Console** — sign in with the environment break-glass
|
|
364
|
+
account (`CATENCE_CONSOLE_USERNAME`/`CATENCE_CONSOLE_PASSWORD_HASH`) and
|
|
365
|
+
restore access with `catence-console users` (reset a password, re-grant
|
|
366
|
+
athletes, or add a new admin).
|
|
@@ -81,14 +81,17 @@ catence
|
|
|
81
81
|
```
|
|
82
82
|
|
|
83
83
|
An agent first calls `list_athletes`, then includes `athleteId` with every
|
|
84
|
-
data tool.
|
|
85
|
-
|
|
84
|
+
data tool. Tools act only on the named athlete store and never merge athletes
|
|
85
|
+
implicitly; the MCP server does not authenticate callers — it trusts the
|
|
86
|
+
`athleteId` each call names.
|
|
86
87
|
|
|
87
88
|
## Multi-athlete catalog
|
|
88
89
|
|
|
89
|
-
Catence
|
|
90
|
-
|
|
91
|
-
athletes
|
|
90
|
+
Catence serves several athlete stores from one server. Each store has its own
|
|
91
|
+
DuckDB, raw data, and tokens; a call acts on the single store it names and
|
|
92
|
+
never merges athletes implicitly. That is a tool contract, not an access
|
|
93
|
+
boundary: the MCP server does not authenticate callers and trusts the
|
|
94
|
+
`athleteId` they pass — see [MCP clients and HTTP](#streamable-http-server).
|
|
92
95
|
|
|
93
96
|
### MCP behavior
|
|
94
97
|
|
|
@@ -300,6 +303,10 @@ By default it is loopback-only. `CATENCE_HTTP_HOST` and `CATENCE_HTTP_PORT`
|
|
|
300
303
|
override the host and port when the flags are omitted.
|
|
301
304
|
|
|
302
305
|
- `GET /api/v1/athletes` returns IDs and labels only.
|
|
306
|
+
- `POST /api/v1/athletes` creates a catalog athlete from
|
|
307
|
+
`{"id": "sam", "label": "Sam", "setDefault": false}` and answers with the
|
|
308
|
+
updated roster; duplicate ids are refused with `409 athlete_exists`, and
|
|
309
|
+
single-store runtimes without a catalog with `400 catalog_required`.
|
|
303
310
|
- `GET /api/v1/dashboard` requires `athleteId`, for example
|
|
304
311
|
`http://127.0.0.1:8787/api/v1/dashboard?athleteId=alex&days=28` (`endDate`
|
|
305
312
|
is an optional `YYYY-MM-DD`; `days` ranges 1-90 and defaults to 28).
|
package/docs/development.md
CHANGED
|
@@ -321,9 +321,11 @@ Recommended dev defaults:
|
|
|
321
321
|
- The deployed scaffold's `sync.sh` / `hash-password.sh` / `doctor.sh` helpers are generated by `scripts/deploy-console.sh`; the dev container does not generate them — use the `npm run` / `uv run` equivalents above, or run the same `catence-data --home /data` commands via `docker compose exec`.
|
|
322
322
|
- If the Console settings panel shows no Model dropdown, the data home's `config.json` has no `console.profiles` — re-copy `config.example.json` or re-run the discovery script, then restart the Console.
|
|
323
323
|
|
|
324
|
-
### Status
|
|
324
|
+
### Settings, Status, and Dashboard
|
|
325
325
|
|
|
326
|
-
|
|
326
|
+
Top-level navigation is **Chat**, **Dashboard**, and **Settings**; the chat-history sidebar only renders on Chat routes, so the other pages use the full width. **Settings** (`/settings`) is a single page whose tabs — Profile, Status, and Models for every account, plus Athletes and Accounts for admins — are deep-linkable as `/settings?tab=<tab>`; asking for a tab the account cannot see falls back to the first visible one.
|
|
327
|
+
|
|
328
|
+
The **Dashboard** (`/dashboard`) is a pure analytics view — it renders charts and recent activities from `GET /api/v1/dashboard` and, when a detached sync holds the DuckDB write lock, shows a non-blocking amber banner (“Data temporarily unavailable — a sync is running. View live progress in Settings → Status.”) instead of a full-page `dashboard_unavailable` error. **Settings → Status** (`/settings?tab=status`) is the lock-safe control plane: it reads `GET /api/v1/health` (runtime `catenceRuntimeHealth()`), `GET /api/v1/sync/status` (sidecar heartbeats + `lastSync`), and `GET /api/v1/athletes` (roster), so it never needs a successful `dashboardSnapshot` and never 400s under lock. The interactive **Sync** button (with instant grey-out via `startingRef`) lives only there; Dashboard keeps a read-only “Syncing…” pill and a “Sync moved to Settings → Status” banner.
|
|
327
329
|
|
|
328
330
|
## Troubleshooting
|
|
329
331
|
|
|
@@ -334,4 +336,4 @@ The **Dashboard** (`/dashboard`) is now a pure analytics view — it renders cha
|
|
|
334
336
|
1. Runtime status API: `curl -s "http://127.0.0.1:8787/api/v1/sync/status?athleteId=<id>" | jq '.progress.running'` (through the Console instead: `/api/v1/sync/status` behind your login).
|
|
335
337
|
2. Detached-child log files: `ls -t "$CATENCE_HOME/logs" | head`. A successful `POST /api/v1/sync` creates `logs/sync-<athlete>-<stamp>.log` before spawning, so an absent or empty `logs/` directory proves no detached sync ever started.
|
|
336
338
|
3. The MCP progress tools (`catence_sync_progress`) read the same heartbeats.
|
|
337
|
-
- **Models page clutter** — unused shipped profiles can be hidden per device from
|
|
339
|
+
- **Models page clutter** — unused shipped profiles can be hidden per device from **Settings → Models** (`Hide` on the profile card); hidden profiles disappear from the chat Model dropdown but stay configured, and are restored with `Unhide`. Hiding lives in the Console database, never in `config.json`.
|
package/docs/llm-providers.md
CHANGED
|
@@ -203,10 +203,10 @@ of the live catalog are pruned from the managed profiles, so the Console never
|
|
|
203
203
|
offers a route the gateway no longer serves; if a pruned model was the
|
|
204
204
|
profile's default, discovery selects a current one instead.
|
|
205
205
|
|
|
206
|
-
You do not have to run the script manually: the
|
|
207
|
-
|
|
208
|
-
these profiles from the live catalog before starting a data sync, and
|
|
209
|
-
|
|
206
|
+
You do not have to run the script manually: the **Sync data** button in
|
|
207
|
+
**Settings → Status** (and `POST /api/v1/sync` with `"refreshModels": true`)
|
|
208
|
+
refreshes these profiles from the live catalog before starting a data sync, and
|
|
209
|
+
the **Discover OpenCode Go models** button in **Settings → Models** (or
|
|
210
210
|
`POST /api/v1/models/discover`) refreshes them without syncing anything.
|
|
211
211
|
Discovery failures — offline machine, catalog unreachable — are reported as a
|
|
212
212
|
warning and never block or fail the sync itself.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "catence",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.1-beta.10",
|
|
4
4
|
"description": "Local MCP access to normalized Garmin, Intervals.icu, and targeted Strava fitness data.",
|
|
5
5
|
"license": "AGPL-3.0-or-later",
|
|
6
6
|
"repository": {
|
|
@@ -55,6 +55,7 @@
|
|
|
55
55
|
"test:watch": "vitest",
|
|
56
56
|
"catence-data": "tsx src/interfaces/cli/main.ts",
|
|
57
57
|
"mcp": "tsx src/interfaces/mcp/main.ts",
|
|
58
|
+
"dev": "node scripts/dev.mjs",
|
|
58
59
|
"console": "uv run --project console catence-console serve",
|
|
59
60
|
"console:doctor": "uv run --project console catence-console doctor",
|
|
60
61
|
"discover:opencode-go": "node scripts/discover-opencode-go.mjs",
|
|
@@ -79,11 +80,12 @@
|
|
|
79
80
|
"@types/node": "^24.3.0",
|
|
80
81
|
"tsx": "^4.20.5",
|
|
81
82
|
"typescript": "^5.9.2",
|
|
82
|
-
"vitest": "^
|
|
83
|
+
"vitest": "^5.0.0"
|
|
83
84
|
},
|
|
84
85
|
"overrides": {
|
|
85
|
-
"fast-uri": "3.1.
|
|
86
|
-
"hono": "4.
|
|
87
|
-
"nanoid": "3.3.18"
|
|
86
|
+
"fast-uri": "3.1.8",
|
|
87
|
+
"hono": "4.13.7",
|
|
88
|
+
"nanoid": "3.3.18",
|
|
89
|
+
"qs": "6.16.0"
|
|
88
90
|
}
|
|
89
91
|
}
|
package/pyproject.toml
CHANGED
|
@@ -15,7 +15,7 @@ dependencies = [
|
|
|
15
15
|
|
|
16
16
|
[tool.uv.sources]
|
|
17
17
|
# fork commit
|
|
18
|
-
garminconnect = { git = "https://github.com/rifusaki/python-garminconnect.git", rev = "
|
|
18
|
+
garminconnect = { git = "https://github.com/rifusaki/python-garminconnect.git", rev = "38b3d5003ae348de384f679976d0527316775f94" }
|
|
19
19
|
|
|
20
20
|
[tool.uv]
|
|
21
21
|
package = false
|
|
@@ -8,6 +8,7 @@ import signal
|
|
|
8
8
|
import sys
|
|
9
9
|
from datetime import date
|
|
10
10
|
from pathlib import Path
|
|
11
|
+
from typing import Any
|
|
11
12
|
|
|
12
13
|
from .progress import ProgressReporter
|
|
13
14
|
from .staging import StagingWriter
|
|
@@ -40,7 +41,31 @@ def parse_args() -> argparse.Namespace:
|
|
|
40
41
|
type=date.fromisoformat,
|
|
41
42
|
help="Fetch calendar events from this date instead of the default short lookback.",
|
|
42
43
|
)
|
|
43
|
-
|
|
44
|
+
parser.add_argument(
|
|
45
|
+
"--concurrency",
|
|
46
|
+
type=int,
|
|
47
|
+
default=5,
|
|
48
|
+
help="Maximum in-flight Garmin requests (1 disables request parallelization).",
|
|
49
|
+
)
|
|
50
|
+
args = parser.parse_args()
|
|
51
|
+
if args.concurrency < 1:
|
|
52
|
+
parser.error("--concurrency must be at least 1")
|
|
53
|
+
return args
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _print_run_summary(run_id: str, worker: GarminStagingWorker, reporter: ProgressReporter) -> None:
|
|
57
|
+
"""Emit one stderr record with per-stage timings and capture counts.
|
|
58
|
+
|
|
59
|
+
The management layer surfaces stderr in the sync log, so quiet-day runs
|
|
60
|
+
carry their own performance profile without an extra instrumentation pass.
|
|
61
|
+
"""
|
|
62
|
+
record: dict[str, Any] = {
|
|
63
|
+
"kind": "stage_timings",
|
|
64
|
+
"runId": run_id,
|
|
65
|
+
"timings": {name: round(seconds, 2) for name, seconds in reporter.stage_summary().items()},
|
|
66
|
+
}
|
|
67
|
+
record.update(worker.capture_summary())
|
|
68
|
+
print(json.dumps(record, separators=(",", ":")), file=sys.stderr, flush=True)
|
|
44
69
|
|
|
45
70
|
|
|
46
71
|
def main() -> None:
|
|
@@ -65,7 +90,7 @@ def main() -> None:
|
|
|
65
90
|
loaded = json.loads(args.known_activities.read_text(encoding="utf-8"))
|
|
66
91
|
if isinstance(loaded, dict):
|
|
67
92
|
known_hashes = {str(key): str(value) for key, value in loaded.items()}
|
|
68
|
-
worker = GarminStagingWorker(client, writer, args.data_dir, known_hashes, progress=reporter)
|
|
93
|
+
worker = GarminStagingWorker(client, writer, args.data_dir, known_hashes, progress=reporter, concurrency=args.concurrency)
|
|
69
94
|
interrupt_signal: int | None = None
|
|
70
95
|
|
|
71
96
|
def _request_interrupt(signum: int, _frame: object) -> None:
|
|
@@ -89,7 +114,10 @@ def main() -> None:
|
|
|
89
114
|
except WorkerInterrupted:
|
|
90
115
|
reporter.finish("interrupted")
|
|
91
116
|
raise SystemExit(130 if interrupt_signal == signal.SIGINT else 143)
|
|
92
|
-
|
|
117
|
+
else:
|
|
118
|
+
reporter.finish("completed")
|
|
119
|
+
finally:
|
|
120
|
+
_print_run_summary(args.run_id, worker, reporter)
|
|
93
121
|
|
|
94
122
|
|
|
95
123
|
if __name__ == "__main__":
|
|
@@ -2,19 +2,26 @@
|
|
|
2
2
|
|
|
3
3
|
Emits single-line JSON progress records (kind: 'progress') on stdout so the
|
|
4
4
|
Node management layer can persist heartbeats without parsing the staging
|
|
5
|
-
JSONL consumed by the importer.
|
|
5
|
+
JSONL consumed by the importer. Captures run on several threads once sync
|
|
6
|
+
concurrency is enabled, so every state mutation happens under a lock while
|
|
7
|
+
the actual emit stays outside it.
|
|
8
|
+
|
|
9
|
+
The reporter also accumulates per-stage wall-clock timings; the CLI prints
|
|
10
|
+
them to stderr as a single `stage_timings` record so each sync run carries
|
|
11
|
+
its own phase profile.
|
|
6
12
|
"""
|
|
7
13
|
|
|
8
14
|
from __future__ import annotations
|
|
9
15
|
|
|
10
16
|
import json
|
|
17
|
+
import threading
|
|
11
18
|
import time
|
|
12
19
|
from datetime import datetime, timezone
|
|
13
20
|
from typing import Any, Callable
|
|
14
21
|
|
|
15
22
|
|
|
16
23
|
class ProgressReporter:
|
|
17
|
-
"""Throttled progress publisher writing NDJSON records to stdout."""
|
|
24
|
+
"""Throttled, thread-safe progress publisher writing NDJSON records to stdout."""
|
|
18
25
|
|
|
19
26
|
def __init__(
|
|
20
27
|
self,
|
|
@@ -27,59 +34,88 @@ class ProgressReporter:
|
|
|
27
34
|
self.provider = provider
|
|
28
35
|
self.emit = emit
|
|
29
36
|
self.interval_seconds = interval_seconds
|
|
37
|
+
self._lock = threading.Lock()
|
|
30
38
|
self._started = time.monotonic()
|
|
31
39
|
self._last_emitted = 0.0
|
|
40
|
+
self._stage_started = self._started
|
|
32
41
|
self.stage = "starting"
|
|
33
42
|
self.current_step: str | None = None
|
|
34
43
|
self.completed_units = 0
|
|
35
44
|
self.total_units: int | None = None
|
|
45
|
+
self.stage_timings: dict[str, float] = {}
|
|
36
46
|
|
|
37
47
|
def set_stage(self, stage: str) -> None:
|
|
38
48
|
"""Switch phases, resetting phase-local counters and forcing a publish."""
|
|
39
|
-
|
|
49
|
+
with self._lock:
|
|
50
|
+
if stage == self.stage:
|
|
51
|
+
return
|
|
52
|
+
self._close_stage_locked()
|
|
40
53
|
self.stage = stage
|
|
41
54
|
self.current_step = None
|
|
42
55
|
self.completed_units = 0
|
|
43
56
|
self.total_units = None
|
|
44
|
-
|
|
57
|
+
self.publish(force=True)
|
|
45
58
|
|
|
46
59
|
def advance(self, completed: int | None = None, total: int | None = None, step: str | None = None) -> None:
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
60
|
+
with self._lock:
|
|
61
|
+
if completed is not None:
|
|
62
|
+
self.completed_units = completed
|
|
63
|
+
if total is not None:
|
|
64
|
+
self.total_units = total
|
|
65
|
+
if step is not None:
|
|
66
|
+
self.current_step = step
|
|
67
|
+
self.publish()
|
|
68
|
+
|
|
69
|
+
def add_completed(self, amount: int = 1) -> None:
|
|
70
|
+
"""Increment the completed-unit counter without replacing it."""
|
|
71
|
+
with self._lock:
|
|
72
|
+
self.completed_units += amount
|
|
53
73
|
self.publish()
|
|
54
74
|
|
|
55
75
|
def publish(self, force: bool = False) -> None:
|
|
56
76
|
now = time.monotonic()
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
77
|
+
with self._lock:
|
|
78
|
+
if not force and now - self._last_emitted < self.interval_seconds:
|
|
79
|
+
return
|
|
80
|
+
self._last_emitted = now
|
|
81
|
+
elapsed = now - self._started
|
|
82
|
+
percent = 0.0
|
|
83
|
+
eta: float | None = None
|
|
84
|
+
if self.total_units and self.total_units > 0:
|
|
85
|
+
percent = min(100.0, max(0.0, self.completed_units / self.total_units * 100.0))
|
|
86
|
+
if 0 < self.completed_units < self.total_units:
|
|
87
|
+
rate = elapsed / self.completed_units
|
|
88
|
+
eta = rate * (self.total_units - self.completed_units)
|
|
89
|
+
record: dict[str, Any] = {
|
|
90
|
+
"kind": "progress",
|
|
91
|
+
"runId": self.run_id,
|
|
92
|
+
"provider": self.provider,
|
|
93
|
+
"stage": self.stage,
|
|
94
|
+
"currentStep": self.current_step,
|
|
95
|
+
"completedUnits": self.completed_units,
|
|
96
|
+
"totalUnits": self.total_units,
|
|
97
|
+
"percentComplete": round(percent, 2),
|
|
98
|
+
"elapsedSeconds": round(elapsed, 1),
|
|
99
|
+
"estimatedRemainingSeconds": round(eta, 1) if eta is not None else None,
|
|
100
|
+
"heartbeatAt": datetime.now(timezone.utc).isoformat(),
|
|
101
|
+
}
|
|
102
|
+
# Emitting outside the lock keeps a slow pipe from stalling captures.
|
|
81
103
|
self.emit(json.dumps(record, separators=(",", ":")))
|
|
82
104
|
|
|
105
|
+
def stage_summary(self) -> dict[str, float]:
|
|
106
|
+
"""Per-stage elapsed seconds, including the still-open stage."""
|
|
107
|
+
with self._lock:
|
|
108
|
+
summary = dict(self.stage_timings)
|
|
109
|
+
summary[self.stage] = summary.get(self.stage, 0.0) + (time.monotonic() - self._stage_started)
|
|
110
|
+
return summary
|
|
111
|
+
|
|
83
112
|
def finish(self, stage: str = "completed") -> None:
|
|
84
|
-
self.
|
|
113
|
+
with self._lock:
|
|
114
|
+
self._close_stage_locked()
|
|
115
|
+
self.stage = stage
|
|
116
|
+
self._stage_started = time.monotonic()
|
|
85
117
|
self.publish(force=True)
|
|
118
|
+
|
|
119
|
+
def _close_stage_locked(self) -> None:
|
|
120
|
+
self.stage_timings[self.stage] = self.stage_timings.get(self.stage, 0.0) + (time.monotonic() - self._stage_started)
|
|
121
|
+
self._stage_started = time.monotonic()
|
|
@@ -3,10 +3,12 @@ from __future__ import annotations
|
|
|
3
3
|
import hashlib
|
|
4
4
|
import json
|
|
5
5
|
import os
|
|
6
|
+
import threading
|
|
6
7
|
from dataclasses import dataclass
|
|
7
8
|
from datetime import datetime, timezone
|
|
8
9
|
from pathlib import Path
|
|
9
10
|
from typing import Any
|
|
11
|
+
from uuid import uuid4
|
|
10
12
|
|
|
11
13
|
from . import STAGING_SCHEMA_VERSION
|
|
12
14
|
from .contracts import validate_record
|
|
@@ -28,12 +30,15 @@ class StagingWriter:
|
|
|
28
30
|
def __post_init__(self) -> None:
|
|
29
31
|
self.output.parent.mkdir(parents=True, exist_ok=True)
|
|
30
32
|
self.output.touch(exist_ok=True)
|
|
33
|
+
self._lock = threading.Lock()
|
|
31
34
|
|
|
32
35
|
def emit(self, record: dict[str, Any]) -> None:
|
|
33
36
|
record["schemaVersion"] = STAGING_SCHEMA_VERSION
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
+
line = json.dumps(validate_record(record), separators=(",", ":"), default=str) + "\n"
|
|
38
|
+
# Sync runs fan capture out across threads; serialize whole lines so the
|
|
39
|
+
# JSONL never interleaves, even though each write still opens the file.
|
|
40
|
+
with self._lock, self.output.open("a", encoding="utf-8") as file:
|
|
41
|
+
file.write(line)
|
|
37
42
|
|
|
38
43
|
def manifest(self, run_id: str, from_date: str) -> None:
|
|
39
44
|
self.emit({
|
|
@@ -54,7 +59,9 @@ class StagingWriter:
|
|
|
54
59
|
destination = self.data_dir / relative
|
|
55
60
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
56
61
|
if not destination.exists():
|
|
57
|
-
|
|
62
|
+
# Unique same-directory temp name: concurrent writers can race on
|
|
63
|
+
# the same content hash, and a shared temp path would clobber.
|
|
64
|
+
temporary = destination.with_name(f".{destination.name}.{os.getpid()}.{uuid4().hex}.tmp")
|
|
58
65
|
temporary.write_bytes(contents)
|
|
59
66
|
os.replace(temporary, destination)
|
|
60
67
|
self.emit({
|