catence 0.2.1-beta.6 → 0.2.1-beta.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/config.example.json +1 -0
- package/dist/contracts/release.d.ts +1 -1
- package/dist/contracts/release.js +1 -1
- package/dist/contracts/runtime.d.ts +4 -0
- package/dist/core/query/catalog.d.ts +1 -1
- package/dist/core/query/catalog.js +16 -10
- package/dist/core/query/repository.d.ts +1 -1
- package/dist/core/query/repository.js +31 -2
- package/dist/core/runtime/athlete-file.d.ts +49 -0
- package/dist/core/runtime/athlete-file.js +177 -0
- package/dist/core/runtime/configuration.js +2 -0
- package/dist/elt/application/management.js +28 -3
- package/dist/elt/normalization/normalizers.js +62 -0
- package/dist/elt/storage/migrations.js +45 -1
- package/dist/interfaces/http/server.js +98 -3
- package/dist/interfaces/mcp/server.js +69 -14
- package/dist/runtime/index.d.ts +2 -0
- package/dist/runtime/index.js +1 -0
- package/docs/configuration.md +47 -0
- package/docs/deployment/docker.md +87 -8
- package/docs/deployment/local-console.md +98 -11
- package/docs/deployment/local-mcp.md +8 -5
- package/package.json +1 -1
- package/pyproject.toml +1 -1
- package/python/catence/providers/garmin/cli.py +31 -3
- package/python/catence/providers/garmin/progress.py +71 -35
- package/python/catence/providers/garmin/staging.py +11 -4
- package/python/catence/providers/garmin/worker.py +420 -206
- package/uv.lock +3 -3
|
@@ -26,6 +26,7 @@ explained in the [beta release notes](../../README.md#beta-releases).
|
|
|
26
26
|
| Provider credentials | per-athlete secret file, mode 0600 | `catence-data secret set` |
|
|
27
27
|
| API keys | process environment only | `.env` |
|
|
28
28
|
| Chat history + preferences | data home | `console/chat-history.sqlite3` |
|
|
29
|
+
| Console logins + athlete grants | data home (mode 0600, bcrypt hashes) | `console/accounts.json` |
|
|
29
30
|
|
|
30
31
|
Credentials never land in `config.json`; profiles reference environment
|
|
31
32
|
variable *names* (`apiKeyEnv`, `apiBaseEnv`, `apiVersionEnv`). See
|
|
@@ -69,10 +70,16 @@ Script options (run `./deploy-console.sh --help` for the full list):
|
|
|
69
70
|
|
|
70
71
|
## 2. Fill in `catence-deploy/.env`
|
|
71
72
|
|
|
73
|
+
The Console always needs `CHAINLIT_AUTH_SECRET`, plus at least one login
|
|
74
|
+
source: the **environment break-glass account**
|
|
75
|
+
(`CATENCE_CONSOLE_USERNAME` + `CATENCE_CONSOLE_PASSWORD_HASH`, set together or
|
|
76
|
+
not at all) or a **stored Console account** in the data volume (see
|
|
77
|
+
[Console accounts](#console-accounts)). Add one model-provider key either way.
|
|
78
|
+
|
|
72
79
|
```sh
|
|
73
|
-
CATENCE_CONSOLE_USERNAME=coach
|
|
74
|
-
CATENCE_CONSOLE_PASSWORD_HASH='…' # generate: ./catence-deploy/hash-password.sh
|
|
75
80
|
CHAINLIT_AUTH_SECRET='…' # generate: openssl rand -hex 32
|
|
81
|
+
CATENCE_CONSOLE_USERNAME=coach # environment break-glass account …
|
|
82
|
+
CATENCE_CONSOLE_PASSWORD_HASH='…' # generate: ./catence-deploy/hash-password.sh
|
|
76
83
|
OPENAI_API_KEY='…' # or ANTHROPIC_API_KEY / OPENCODE_GO_API_KEY …
|
|
77
84
|
OPENAI_API_BASE='' # only for OpenAI-compatible endpoints
|
|
78
85
|
OPENCODE_GO_API_KEY='…' # the OpenCode Go base URLs are pre-filled below
|
|
@@ -80,9 +87,14 @@ OPENCODE_GO_API_BASE='https://opencode.ai/zen/go/v1'
|
|
|
80
87
|
OPENCODE_GO_MESSAGES_API_BASE='https://opencode.ai/zen/go'
|
|
81
88
|
```
|
|
82
89
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
90
|
+
Leave the username/hash pair empty to log in with stored accounts instead: the
|
|
91
|
+
script accepts an existing `console/accounts.json` in the data volume and
|
|
92
|
+
starts without the pair. Set both or neither — a lone username without a hash
|
|
93
|
+
fails the deploy with a clear error.
|
|
94
|
+
|
|
95
|
+
Generate the hash for the environment pair with the scaffold helper (it
|
|
96
|
+
prompts on the terminal, never echoes, and uses the compose project so it works
|
|
97
|
+
whatever image tag the script built):
|
|
86
98
|
|
|
87
99
|
```sh
|
|
88
100
|
./catence-deploy/hash-password.sh
|
|
@@ -102,6 +114,66 @@ Then start the stack (or re-run the script, which re-reads the saved `.env`):
|
|
|
102
114
|
docker compose -f catence-deploy/docker-compose.yml --env-file catence-deploy/.env up -d
|
|
103
115
|
```
|
|
104
116
|
|
|
117
|
+
### Console accounts
|
|
118
|
+
|
|
119
|
+
Stored accounts are the multi-user path: each account has a role and an
|
|
120
|
+
athlete grant list, and the Console enforces both server-side on logins,
|
|
121
|
+
roster/dashboard requests, and agent turns. Accounts live in
|
|
122
|
+
`console/accounts.json` in the data volume (mode 0600, bcrypt hashes only).
|
|
123
|
+
|
|
124
|
+
Manage them with the `users` command inside the container — via the generated
|
|
125
|
+
helper:
|
|
126
|
+
|
|
127
|
+
```sh
|
|
128
|
+
./catence-deploy/users.sh add martina --athlete martina # member: one granted athlete
|
|
129
|
+
./catence-deploy/users.sh add coach --admin # admin: every athlete
|
|
130
|
+
./catence-deploy/users.sh list
|
|
131
|
+
./catence-deploy/users.sh grant martina sam
|
|
132
|
+
./catence-deploy/users.sh revoke martina sam
|
|
133
|
+
./catence-deploy/users.sh passwd martina
|
|
134
|
+
./catence-deploy/users.sh set-role martina admin
|
|
135
|
+
./catence-deploy/users.sh remove martina
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
`users.sh` uses `docker compose exec`, so the stack must be running. The long
|
|
139
|
+
forms are:
|
|
140
|
+
|
|
141
|
+
```sh
|
|
142
|
+
# Interactive: prompts for the password twice
|
|
143
|
+
docker compose -f catence-deploy/docker-compose.yml --env-file catence-deploy/.env exec console \
|
|
144
|
+
/opt/catence-console/bin/catence-console users add martina --athlete martina
|
|
145
|
+
|
|
146
|
+
# Non-interactive: read the password from an environment variable
|
|
147
|
+
docker compose -f catence-deploy/docker-compose.yml --env-file catence-deploy/.env run --rm -T \
|
|
148
|
+
-e CATENCE_CONSOLE_NEW_PASSWORD='…' \
|
|
149
|
+
--entrypoint /opt/catence-console/bin/catence-console console \
|
|
150
|
+
users add martina --athlete martina --password-env CATENCE_CONSOLE_NEW_PASSWORD
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
Before the first start there is no running container **and** no login yet —
|
|
154
|
+
the Console refuses to start without one — so create the first account with
|
|
155
|
+
the one-off form (it prompts for the password):
|
|
156
|
+
|
|
157
|
+
```sh
|
|
158
|
+
docker compose -f catence-deploy/docker-compose.yml --env-file catence-deploy/.env run --rm \
|
|
159
|
+
--entrypoint /opt/catence-console/bin/catence-console console users add coach --admin
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
Roles: `admin` sees every athlete; `member` sees only explicitly granted
|
|
163
|
+
athletes. A member with zero grants can sign in and browse, but has no athlete
|
|
164
|
+
to scope a chat to. The environment pair is a never-persisted break-glass
|
|
165
|
+
admin: it keeps working when the accounts file is lost or the last admin is
|
|
166
|
+
locked out, and unsetting the two variables revokes it.
|
|
167
|
+
|
|
168
|
+
Accounts gate the Console only. The MCP server keeps its own contract — no
|
|
169
|
+
authentication, the caller's `athleteId` is trusted — so port 8787 still needs
|
|
170
|
+
loopback or network-layer restriction (see
|
|
171
|
+
[Connecting to the Docker-based MCP from outside](#connecting-to-the-docker-based-mcp-from-outside)).
|
|
172
|
+
|
|
173
|
+
The accounts file is the only copy of your stored logins: keep it in backups
|
|
174
|
+
of the data volume (or bind-mounted home), along with `config.json` and the
|
|
175
|
+
athlete stores.
|
|
176
|
+
|
|
105
177
|
## 3. Initialize an athlete store
|
|
106
178
|
|
|
107
179
|
The data volume already contains a seeded `config.json`, so `setup` accepts
|
|
@@ -184,8 +256,9 @@ export OPENCODE_GO_API_KEY='…' # any non-empty value pass
|
|
|
184
256
|
## 7. Access the Console
|
|
185
257
|
|
|
186
258
|
Open `http://127.0.0.1:8000` (or the `--bind:--port` you configured) and log in
|
|
187
|
-
with
|
|
188
|
-
`
|
|
259
|
+
with a stored Console account (see [Console accounts](#console-accounts)) or
|
|
260
|
+
with the environment break-glass pair (`CATENCE_CONSOLE_USERNAME` and the
|
|
261
|
+
password whose hash is in `CATENCE_CONSOLE_PASSWORD_HASH`).
|
|
189
262
|
|
|
190
263
|
## 8. Verify with doctor
|
|
191
264
|
|
|
@@ -472,11 +545,17 @@ docker run --rm -it -v catence-data:/data --entrypoint catence-data catence-cons
|
|
|
472
545
|
|
|
473
546
|
Use the same `--entrypoint catence-data` pattern for `secret set` and `sync`.
|
|
474
547
|
|
|
548
|
+
Console logins work the same way: keep the environment pair together (or
|
|
549
|
+
neither variable), or seed a stored account into the volume with the `users`
|
|
550
|
+
command (see [Console accounts](#console-accounts)).
|
|
551
|
+
|
|
475
552
|
## Updating
|
|
476
553
|
|
|
477
554
|
Docker deployments: re-run `deploy-console.sh beta` (or `stable`); it
|
|
478
555
|
re-resolves the newest versions from the registries and rebuilds. The data
|
|
479
|
-
volume
|
|
556
|
+
volume is preserved, so your `config.json`, the stored Console accounts
|
|
557
|
+
(`console/accounts.json`), chat history, and the athlete stores all survive
|
|
558
|
+
the rebuild.
|
|
480
559
|
|
|
481
560
|
```sh
|
|
482
561
|
curl -fsSL https://raw.githubusercontent.com/rifusaki/catence/beta/scripts/deploy-console.sh | bash -s beta
|
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
Catence Console is a password-protected local web chat that starts a matching
|
|
4
4
|
Catence runtime on loopback. It uses Chainlit password login and fails closed
|
|
5
|
-
unless
|
|
5
|
+
unless `CHAINLIT_AUTH_SECRET` and at least one login source — a stored Console
|
|
6
|
+
account or the environment account pair — are configured.
|
|
6
7
|
|
|
7
8
|
This guide covers the **local** path: the npm package plus the Python Console
|
|
8
9
|
on your machine. The Docker path (one container built from the public
|
|
@@ -27,6 +28,7 @@ model.
|
|
|
27
28
|
| Provider credentials | per-athlete secret file, mode 0600 | `catence-data secret set` |
|
|
28
29
|
| API keys | process environment only | shell exports |
|
|
29
30
|
| Chat history + preferences | data home | `console/chat-history.sqlite3` |
|
|
31
|
+
| Console logins + athlete grants | data home (mode 0600, bcrypt hashes) | `console/accounts.json` |
|
|
30
32
|
|
|
31
33
|
Credentials never land in `config.json`; profiles reference environment
|
|
32
34
|
variable *names* (`apiKeyEnv`, `apiBaseEnv`, `apiVersionEnv`). See
|
|
@@ -95,15 +97,28 @@ To add OpenCode Go models, run the
|
|
|
95
97
|
|
|
96
98
|
```sh
|
|
97
99
|
export OPENAI_API_KEY='…' # or ANTHROPIC_API_KEY / OPENCODE_GO_API_KEY …
|
|
98
|
-
export CATENCE_CONSOLE_USERNAME='coach'
|
|
99
|
-
export CATENCE_CONSOLE_PASSWORD_HASH="$(catence-console auth hash-password)"
|
|
100
100
|
export CHAINLIT_AUTH_SECRET="$(openssl rand -hex 32)"
|
|
101
101
|
```
|
|
102
102
|
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
103
|
+
`CHAINLIT_AUTH_SECRET` is always required: it signs Chainlit sessions. On top
|
|
104
|
+
of it, the Console needs at least one login source:
|
|
105
|
+
|
|
106
|
+
- **Stored accounts** (the multi-user path). Each account carries a role and
|
|
107
|
+
an athlete grant list and lives in `<home>/console/accounts.json`. Create
|
|
108
|
+
the first one with the users CLI, which prompts for the password twice:
|
|
109
|
+
|
|
110
|
+
```sh
|
|
111
|
+
catence-console users add coach --admin
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
- **The environment break-glass account**: set both
|
|
115
|
+
`CATENCE_CONSOLE_USERNAME` and `CATENCE_CONSOLE_PASSWORD_HASH` (the bcrypt
|
|
116
|
+
hash printed by `catence-console auth hash-password`). The pair is never
|
|
117
|
+
persisted and always acts as an admin — the way back in when the accounts
|
|
118
|
+
file is lost.
|
|
119
|
+
|
|
120
|
+
Roles, grants, and the rest of the CLI are described under
|
|
121
|
+
[Console accounts](#console-accounts).
|
|
107
122
|
|
|
108
123
|
## 7. Start the Console
|
|
109
124
|
|
|
@@ -143,6 +158,68 @@ origin, shows live progress while the run is active, and displays the last
|
|
|
143
158
|
completed sync afterwards. Each manual sync also refreshes OpenCode Go model
|
|
144
159
|
profiles first; a discovery failure never blocks the data sync.
|
|
145
160
|
|
|
161
|
+
## Console accounts
|
|
162
|
+
|
|
163
|
+
Console accounts are Catence's multi-user login model, enforced entirely by
|
|
164
|
+
the Console: sign-in, the athlete roster, dashboard and athlete-file requests,
|
|
165
|
+
and the athlete scoping of every chat turn are all checked server-side against
|
|
166
|
+
the account's role and athlete grants. The MCP server itself is untouched — it
|
|
167
|
+
has no authentication and trusts the `athleteId` each call names (see
|
|
168
|
+
[local-mcp.md](local-mcp.md#streamable-http-server)); accounts only gate what
|
|
169
|
+
goes through the Console.
|
|
170
|
+
|
|
171
|
+
| Role | Athlete access |
|
|
172
|
+
| --- | --- |
|
|
173
|
+
| `admin` | Every athlete in the catalog (`athletes: "all"`) |
|
|
174
|
+
| `member` | Only the explicitly granted athletes; zero grants allowed |
|
|
175
|
+
|
|
176
|
+
Accounts live in `<home>/console/accounts.json` (mode 0600, bcrypt hashes
|
|
177
|
+
only), where `<home>` is `$CATENCE_HOME` or `~/.catence`. The Console re-reads
|
|
178
|
+
the file on every login, so new accounts, password resets, and grant changes
|
|
179
|
+
apply without a restart.
|
|
180
|
+
|
|
181
|
+
### Manage accounts with the users CLI
|
|
182
|
+
|
|
183
|
+
```sh
|
|
184
|
+
catence-console users add coach --admin # admin: all athletes
|
|
185
|
+
catence-console users add martina --athlete martina # member with one grant
|
|
186
|
+
catence-console users add sam # member, zero grants
|
|
187
|
+
catence-console users list
|
|
188
|
+
catence-console users passwd martina
|
|
189
|
+
catence-console users set-role sam member
|
|
190
|
+
catence-console users grant sam martina
|
|
191
|
+
catence-console users revoke sam martina
|
|
192
|
+
catence-console users remove sam
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
- `add <username> (--admin | --athlete <id> [--athlete <id> …])` — members
|
|
196
|
+
default to zero grants; `--admin` cannot be combined with `--athlete` and
|
|
197
|
+
always stores access to every athlete.
|
|
198
|
+
- `add` and `passwd` prompt for the password twice (never echoed). For
|
|
199
|
+
automation, `--password-env VAR` reads the password from environment
|
|
200
|
+
variable `VAR` instead, so it never appears in the command line or shell
|
|
201
|
+
history.
|
|
202
|
+
- `grant`/`revoke` change a member's athlete list; revoking the last grant
|
|
203
|
+
leaves a zero-grant member. `set-role` promotes or demotes: promoting to
|
|
204
|
+
`admin` coerces the grants to `"all"`, demoting to `member` keeps them.
|
|
205
|
+
- `add`/`grant`/`revoke` validate athlete ids against `catalog.json` when it
|
|
206
|
+
exists, and `--home <dir>` points any subcommand at a non-default catalog.
|
|
207
|
+
|
|
208
|
+
A member with zero grants can sign in and browse the Console, but there is no
|
|
209
|
+
athlete to scope a chat to: the Athlete selector is disabled with *"You don't
|
|
210
|
+
have access to any athlete yet. Ask an administrator to grant you access."*
|
|
211
|
+
and every chat turn is answered with the same notice. A member that requests
|
|
212
|
+
an athlete outside their grants receives `403 athlete_forbidden`; the Console
|
|
213
|
+
never forwards the request to the runtime.
|
|
214
|
+
|
|
215
|
+
### The environment break-glass account
|
|
216
|
+
|
|
217
|
+
`CATENCE_CONSOLE_USERNAME` and `CATENCE_CONSOLE_PASSWORD_HASH` must be set
|
|
218
|
+
together. When they are, the pair always acts as an `admin` and is never
|
|
219
|
+
written to `accounts.json`: unsetting the two variables revokes it. Use it as
|
|
220
|
+
the recovery path when the accounts file is lost or a lockout leaves nobody
|
|
221
|
+
who can grant access — not as the everyday login.
|
|
222
|
+
|
|
146
223
|
## Console serve options
|
|
147
224
|
|
|
148
225
|
`catence-console serve` accepts:
|
|
@@ -167,12 +244,18 @@ chat starts.
|
|
|
167
244
|
|
|
168
245
|
## How multi-athlete works in the Console
|
|
169
246
|
|
|
170
|
-
The Console shares the same per-athlete stores as MCP
|
|
171
|
-
|
|
247
|
+
The Console shares the same per-athlete stores as MCP. Access is layered:
|
|
248
|
+
per-account grants at the login, then server-owned scoping per chat.
|
|
172
249
|
|
|
250
|
+
- **Per-account grants.** Each stored account is an `admin` (every athlete) or
|
|
251
|
+
a `member` with an explicit grant list (see
|
|
252
|
+
[Console accounts](#console-accounts)). The **Athlete** selector and the
|
|
253
|
+
proxied roster are narrowed to the account's grants; requests that name an
|
|
254
|
+
ungranted athlete are refused with `403 athlete_forbidden` before they reach
|
|
255
|
+
the runtime, and a zero-grant member sees no athletes at all.
|
|
173
256
|
- The settings panel has an **Athlete** selector built from
|
|
174
257
|
`GET /api/v1/athletes` on the runtime. The default is the catalog's
|
|
175
|
-
`defaultAthleteId
|
|
258
|
+
`defaultAthleteId` (for a member, the first granted athlete).
|
|
176
259
|
- On every personal-data tool call, the Console **forces** the selected
|
|
177
260
|
athleteId onto the arguments — the model cannot name or switch athletes, and
|
|
178
261
|
a chat's system message states: *"This Console chat is scoped to athleteId X.
|
|
@@ -260,4 +343,8 @@ uv tool install --upgrade catence-console
|
|
|
260
343
|
environment variables; set them in the shell and restart the Console.
|
|
261
344
|
- **Runtime not reachable** — `catence-console doctor --home "$HOME/.catence"
|
|
262
345
|
--mcp-url http://127.0.0.1:8787/mcp` checks the handshake; confirm the
|
|
263
|
-
runtime is running on that port and the protocol versions agree.
|
|
346
|
+
runtime is running on that port and the protocol versions agree.
|
|
347
|
+
- **Locked out of the Console** — sign in with the environment break-glass
|
|
348
|
+
account (`CATENCE_CONSOLE_USERNAME`/`CATENCE_CONSOLE_PASSWORD_HASH`) and
|
|
349
|
+
restore access with `catence-console users` (reset a password, re-grant
|
|
350
|
+
athletes, or add a new admin).
|
|
@@ -81,14 +81,17 @@ catence
|
|
|
81
81
|
```
|
|
82
82
|
|
|
83
83
|
An agent first calls `list_athletes`, then includes `athleteId` with every
|
|
84
|
-
data tool.
|
|
85
|
-
|
|
84
|
+
data tool. Tools act only on the named athlete store and never merge athletes
|
|
85
|
+
implicitly; the MCP server does not authenticate callers — it trusts the
|
|
86
|
+
`athleteId` each call names.
|
|
86
87
|
|
|
87
88
|
## Multi-athlete catalog
|
|
88
89
|
|
|
89
|
-
Catence
|
|
90
|
-
|
|
91
|
-
athletes
|
|
90
|
+
Catence serves several athlete stores from one server. Each store has its own
|
|
91
|
+
DuckDB, raw data, and tokens; a call acts on the single store it names and
|
|
92
|
+
never merges athletes implicitly. That is a tool contract, not an access
|
|
93
|
+
boundary: the MCP server does not authenticate callers and trusts the
|
|
94
|
+
`athleteId` they pass — see [MCP clients and HTTP](#streamable-http-server).
|
|
92
95
|
|
|
93
96
|
### MCP behavior
|
|
94
97
|
|
package/package.json
CHANGED
package/pyproject.toml
CHANGED
|
@@ -15,7 +15,7 @@ dependencies = [
|
|
|
15
15
|
|
|
16
16
|
[tool.uv.sources]
|
|
17
17
|
# fork commit
|
|
18
|
-
garminconnect = { git = "https://github.com/rifusaki/python-garminconnect.git", rev = "
|
|
18
|
+
garminconnect = { git = "https://github.com/rifusaki/python-garminconnect.git", rev = "38b3d5003ae348de384f679976d0527316775f94" }
|
|
19
19
|
|
|
20
20
|
[tool.uv]
|
|
21
21
|
package = false
|
|
@@ -8,6 +8,7 @@ import signal
|
|
|
8
8
|
import sys
|
|
9
9
|
from datetime import date
|
|
10
10
|
from pathlib import Path
|
|
11
|
+
from typing import Any
|
|
11
12
|
|
|
12
13
|
from .progress import ProgressReporter
|
|
13
14
|
from .staging import StagingWriter
|
|
@@ -40,7 +41,31 @@ def parse_args() -> argparse.Namespace:
|
|
|
40
41
|
type=date.fromisoformat,
|
|
41
42
|
help="Fetch calendar events from this date instead of the default short lookback.",
|
|
42
43
|
)
|
|
43
|
-
|
|
44
|
+
parser.add_argument(
|
|
45
|
+
"--concurrency",
|
|
46
|
+
type=int,
|
|
47
|
+
default=5,
|
|
48
|
+
help="Maximum in-flight Garmin requests (1 disables request parallelization).",
|
|
49
|
+
)
|
|
50
|
+
args = parser.parse_args()
|
|
51
|
+
if args.concurrency < 1:
|
|
52
|
+
parser.error("--concurrency must be at least 1")
|
|
53
|
+
return args
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _print_run_summary(run_id: str, worker: GarminStagingWorker, reporter: ProgressReporter) -> None:
|
|
57
|
+
"""Emit one stderr record with per-stage timings and capture counts.
|
|
58
|
+
|
|
59
|
+
The management layer surfaces stderr in the sync log, so quiet-day runs
|
|
60
|
+
carry their own performance profile without an extra instrumentation pass.
|
|
61
|
+
"""
|
|
62
|
+
record: dict[str, Any] = {
|
|
63
|
+
"kind": "stage_timings",
|
|
64
|
+
"runId": run_id,
|
|
65
|
+
"timings": {name: round(seconds, 2) for name, seconds in reporter.stage_summary().items()},
|
|
66
|
+
}
|
|
67
|
+
record.update(worker.capture_summary())
|
|
68
|
+
print(json.dumps(record, separators=(",", ":")), file=sys.stderr, flush=True)
|
|
44
69
|
|
|
45
70
|
|
|
46
71
|
def main() -> None:
|
|
@@ -65,7 +90,7 @@ def main() -> None:
|
|
|
65
90
|
loaded = json.loads(args.known_activities.read_text(encoding="utf-8"))
|
|
66
91
|
if isinstance(loaded, dict):
|
|
67
92
|
known_hashes = {str(key): str(value) for key, value in loaded.items()}
|
|
68
|
-
worker = GarminStagingWorker(client, writer, args.data_dir, known_hashes, progress=reporter)
|
|
93
|
+
worker = GarminStagingWorker(client, writer, args.data_dir, known_hashes, progress=reporter, concurrency=args.concurrency)
|
|
69
94
|
interrupt_signal: int | None = None
|
|
70
95
|
|
|
71
96
|
def _request_interrupt(signum: int, _frame: object) -> None:
|
|
@@ -89,7 +114,10 @@ def main() -> None:
|
|
|
89
114
|
except WorkerInterrupted:
|
|
90
115
|
reporter.finish("interrupted")
|
|
91
116
|
raise SystemExit(130 if interrupt_signal == signal.SIGINT else 143)
|
|
92
|
-
|
|
117
|
+
else:
|
|
118
|
+
reporter.finish("completed")
|
|
119
|
+
finally:
|
|
120
|
+
_print_run_summary(args.run_id, worker, reporter)
|
|
93
121
|
|
|
94
122
|
|
|
95
123
|
if __name__ == "__main__":
|
|
@@ -2,19 +2,26 @@
|
|
|
2
2
|
|
|
3
3
|
Emits single-line JSON progress records (kind: 'progress') on stdout so the
|
|
4
4
|
Node management layer can persist heartbeats without parsing the staging
|
|
5
|
-
JSONL consumed by the importer.
|
|
5
|
+
JSONL consumed by the importer. Captures run on several threads once sync
|
|
6
|
+
concurrency is enabled, so every state mutation happens under a lock while
|
|
7
|
+
the actual emit stays outside it.
|
|
8
|
+
|
|
9
|
+
The reporter also accumulates per-stage wall-clock timings; the CLI prints
|
|
10
|
+
them to stderr as a single `stage_timings` record so each sync run carries
|
|
11
|
+
its own phase profile.
|
|
6
12
|
"""
|
|
7
13
|
|
|
8
14
|
from __future__ import annotations
|
|
9
15
|
|
|
10
16
|
import json
|
|
17
|
+
import threading
|
|
11
18
|
import time
|
|
12
19
|
from datetime import datetime, timezone
|
|
13
20
|
from typing import Any, Callable
|
|
14
21
|
|
|
15
22
|
|
|
16
23
|
class ProgressReporter:
|
|
17
|
-
"""Throttled progress publisher writing NDJSON records to stdout."""
|
|
24
|
+
"""Throttled, thread-safe progress publisher writing NDJSON records to stdout."""
|
|
18
25
|
|
|
19
26
|
def __init__(
|
|
20
27
|
self,
|
|
@@ -27,59 +34,88 @@ class ProgressReporter:
|
|
|
27
34
|
self.provider = provider
|
|
28
35
|
self.emit = emit
|
|
29
36
|
self.interval_seconds = interval_seconds
|
|
37
|
+
self._lock = threading.Lock()
|
|
30
38
|
self._started = time.monotonic()
|
|
31
39
|
self._last_emitted = 0.0
|
|
40
|
+
self._stage_started = self._started
|
|
32
41
|
self.stage = "starting"
|
|
33
42
|
self.current_step: str | None = None
|
|
34
43
|
self.completed_units = 0
|
|
35
44
|
self.total_units: int | None = None
|
|
45
|
+
self.stage_timings: dict[str, float] = {}
|
|
36
46
|
|
|
37
47
|
def set_stage(self, stage: str) -> None:
|
|
38
48
|
"""Switch phases, resetting phase-local counters and forcing a publish."""
|
|
39
|
-
|
|
49
|
+
with self._lock:
|
|
50
|
+
if stage == self.stage:
|
|
51
|
+
return
|
|
52
|
+
self._close_stage_locked()
|
|
40
53
|
self.stage = stage
|
|
41
54
|
self.current_step = None
|
|
42
55
|
self.completed_units = 0
|
|
43
56
|
self.total_units = None
|
|
44
|
-
|
|
57
|
+
self.publish(force=True)
|
|
45
58
|
|
|
46
59
|
def advance(self, completed: int | None = None, total: int | None = None, step: str | None = None) -> None:
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
60
|
+
with self._lock:
|
|
61
|
+
if completed is not None:
|
|
62
|
+
self.completed_units = completed
|
|
63
|
+
if total is not None:
|
|
64
|
+
self.total_units = total
|
|
65
|
+
if step is not None:
|
|
66
|
+
self.current_step = step
|
|
67
|
+
self.publish()
|
|
68
|
+
|
|
69
|
+
def add_completed(self, amount: int = 1) -> None:
|
|
70
|
+
"""Increment the completed-unit counter without replacing it."""
|
|
71
|
+
with self._lock:
|
|
72
|
+
self.completed_units += amount
|
|
53
73
|
self.publish()
|
|
54
74
|
|
|
55
75
|
def publish(self, force: bool = False) -> None:
|
|
56
76
|
now = time.monotonic()
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
77
|
+
with self._lock:
|
|
78
|
+
if not force and now - self._last_emitted < self.interval_seconds:
|
|
79
|
+
return
|
|
80
|
+
self._last_emitted = now
|
|
81
|
+
elapsed = now - self._started
|
|
82
|
+
percent = 0.0
|
|
83
|
+
eta: float | None = None
|
|
84
|
+
if self.total_units and self.total_units > 0:
|
|
85
|
+
percent = min(100.0, max(0.0, self.completed_units / self.total_units * 100.0))
|
|
86
|
+
if 0 < self.completed_units < self.total_units:
|
|
87
|
+
rate = elapsed / self.completed_units
|
|
88
|
+
eta = rate * (self.total_units - self.completed_units)
|
|
89
|
+
record: dict[str, Any] = {
|
|
90
|
+
"kind": "progress",
|
|
91
|
+
"runId": self.run_id,
|
|
92
|
+
"provider": self.provider,
|
|
93
|
+
"stage": self.stage,
|
|
94
|
+
"currentStep": self.current_step,
|
|
95
|
+
"completedUnits": self.completed_units,
|
|
96
|
+
"totalUnits": self.total_units,
|
|
97
|
+
"percentComplete": round(percent, 2),
|
|
98
|
+
"elapsedSeconds": round(elapsed, 1),
|
|
99
|
+
"estimatedRemainingSeconds": round(eta, 1) if eta is not None else None,
|
|
100
|
+
"heartbeatAt": datetime.now(timezone.utc).isoformat(),
|
|
101
|
+
}
|
|
102
|
+
# Emitting outside the lock keeps a slow pipe from stalling captures.
|
|
81
103
|
self.emit(json.dumps(record, separators=(",", ":")))
|
|
82
104
|
|
|
105
|
+
def stage_summary(self) -> dict[str, float]:
|
|
106
|
+
"""Per-stage elapsed seconds, including the still-open stage."""
|
|
107
|
+
with self._lock:
|
|
108
|
+
summary = dict(self.stage_timings)
|
|
109
|
+
summary[self.stage] = summary.get(self.stage, 0.0) + (time.monotonic() - self._stage_started)
|
|
110
|
+
return summary
|
|
111
|
+
|
|
83
112
|
def finish(self, stage: str = "completed") -> None:
|
|
84
|
-
self.
|
|
113
|
+
with self._lock:
|
|
114
|
+
self._close_stage_locked()
|
|
115
|
+
self.stage = stage
|
|
116
|
+
self._stage_started = time.monotonic()
|
|
85
117
|
self.publish(force=True)
|
|
118
|
+
|
|
119
|
+
def _close_stage_locked(self) -> None:
|
|
120
|
+
self.stage_timings[self.stage] = self.stage_timings.get(self.stage, 0.0) + (time.monotonic() - self._stage_started)
|
|
121
|
+
self._stage_started = time.monotonic()
|
|
@@ -3,10 +3,12 @@ from __future__ import annotations
|
|
|
3
3
|
import hashlib
|
|
4
4
|
import json
|
|
5
5
|
import os
|
|
6
|
+
import threading
|
|
6
7
|
from dataclasses import dataclass
|
|
7
8
|
from datetime import datetime, timezone
|
|
8
9
|
from pathlib import Path
|
|
9
10
|
from typing import Any
|
|
11
|
+
from uuid import uuid4
|
|
10
12
|
|
|
11
13
|
from . import STAGING_SCHEMA_VERSION
|
|
12
14
|
from .contracts import validate_record
|
|
@@ -28,12 +30,15 @@ class StagingWriter:
|
|
|
28
30
|
def __post_init__(self) -> None:
|
|
29
31
|
self.output.parent.mkdir(parents=True, exist_ok=True)
|
|
30
32
|
self.output.touch(exist_ok=True)
|
|
33
|
+
self._lock = threading.Lock()
|
|
31
34
|
|
|
32
35
|
def emit(self, record: dict[str, Any]) -> None:
|
|
33
36
|
record["schemaVersion"] = STAGING_SCHEMA_VERSION
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
+
line = json.dumps(validate_record(record), separators=(",", ":"), default=str) + "\n"
|
|
38
|
+
# Sync runs fan capture out across threads; serialize whole lines so the
|
|
39
|
+
# JSONL never interleaves, even though each write still opens the file.
|
|
40
|
+
with self._lock, self.output.open("a", encoding="utf-8") as file:
|
|
41
|
+
file.write(line)
|
|
37
42
|
|
|
38
43
|
def manifest(self, run_id: str, from_date: str) -> None:
|
|
39
44
|
self.emit({
|
|
@@ -54,7 +59,9 @@ class StagingWriter:
|
|
|
54
59
|
destination = self.data_dir / relative
|
|
55
60
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
56
61
|
if not destination.exists():
|
|
57
|
-
|
|
62
|
+
# Unique same-directory temp name: concurrent writers can race on
|
|
63
|
+
# the same content hash, and a shared temp path would clobber.
|
|
64
|
+
temporary = destination.with_name(f".{destination.name}.{os.getpid()}.{uuid4().hex}.tmp")
|
|
58
65
|
temporary.write_bytes(contents)
|
|
59
66
|
os.replace(temporary, destination)
|
|
60
67
|
self.emit({
|