catence 0.0.0 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/README.md +373 -29
- package/config.example.json +63 -0
- package/dist/contracts/index.d.ts +4 -0
- package/dist/contracts/index.js +5 -0
- package/dist/contracts/release.d.ts +20 -0
- package/dist/contracts/release.js +17 -0
- package/dist/contracts/runtime.d.ts +11 -0
- package/dist/contracts/runtime.js +2 -0
- package/dist/contracts/staging.d.ts +183 -0
- package/dist/contracts/staging.js +36 -0
- package/dist/contracts/storage.d.ts +15 -0
- package/dist/contracts/storage.js +2 -0
- package/dist/core/dashboard/snapshot.d.ts +16 -0
- package/dist/core/dashboard/snapshot.js +65 -0
- package/dist/core/query/activity-discovery.d.ts +21 -0
- package/dist/core/query/activity-discovery.js +133 -0
- package/dist/core/query/analytics.d.ts +59 -0
- package/dist/core/query/analytics.js +395 -0
- package/dist/core/query/catalog.d.ts +26 -0
- package/dist/core/query/catalog.js +141 -0
- package/dist/core/query/fitness.d.ts +40 -0
- package/dist/core/query/fitness.js +297 -0
- package/dist/core/query/repository.d.ts +19 -0
- package/dist/core/query/repository.js +182 -0
- package/dist/core/query/sql-guard.d.ts +7 -0
- package/dist/core/query/sql-guard.js +121 -0
- package/dist/core/query/sql.d.ts +2 -0
- package/dist/core/query/sql.js +5 -0
- package/dist/core/query/swimming.d.ts +19 -0
- package/dist/core/query/swimming.js +184 -0
- package/dist/core/query/wellness.d.ts +45 -0
- package/dist/core/query/wellness.js +292 -0
- package/dist/core/retrieval/index.d.ts +13 -0
- package/dist/core/retrieval/index.js +248 -0
- package/dist/core/runtime/configuration.d.ts +71 -0
- package/dist/core/runtime/configuration.js +106 -0
- package/dist/core/runtime/limiter.d.ts +13 -0
- package/dist/core/runtime/limiter.js +17 -0
- package/dist/elt/application/demo.d.ts +23 -0
- package/dist/elt/application/demo.js +165 -0
- package/dist/elt/application/management.d.ts +25 -0
- package/dist/elt/application/management.js +283 -0
- package/dist/elt/ingestion/importer.d.ts +8 -0
- package/dist/elt/ingestion/importer.js +87 -0
- package/dist/elt/ingestion/providers/intervals.d.ts +21 -0
- package/dist/elt/ingestion/providers/intervals.js +224 -0
- package/dist/elt/ingestion/providers/registry.d.ts +19 -0
- package/dist/elt/ingestion/providers/registry.js +62 -0
- package/dist/elt/ingestion/providers/strava/service.d.ts +42 -0
- package/dist/elt/ingestion/providers/strava/service.js +365 -0
- package/dist/elt/normalization/activities/linking.d.ts +5 -0
- package/dist/elt/normalization/activities/linking.js +165 -0
- package/dist/elt/normalization/normalizers.d.ts +5 -0
- package/dist/elt/normalization/normalizers.js +816 -0
- package/dist/elt/normalization/segments/strava-normalizer.d.ts +3 -0
- package/dist/elt/normalization/segments/strava-normalizer.js +93 -0
- package/dist/elt/storage/artifacts.d.ts +13 -0
- package/dist/elt/storage/artifacts.js +48 -0
- package/dist/elt/storage/database.d.ts +74 -0
- package/dist/elt/storage/database.js +296 -0
- package/dist/elt/storage/migrations.d.ts +3 -0
- package/dist/elt/storage/migrations.js +832 -0
- package/dist/elt/storage/migrations.legacy.d.ts +12 -0
- package/dist/elt/storage/migrations.legacy.js +332 -0
- package/dist/elt/storage/sql.d.ts +2 -0
- package/dist/elt/storage/sql.js +7 -0
- package/dist/elt/storage/write-lock.d.ts +6 -0
- package/dist/elt/storage/write-lock.js +28 -0
- package/dist/elt/streams.d.ts +24 -0
- package/dist/elt/streams.js +62 -0
- package/dist/interfaces/cli/commands.d.ts +2 -0
- package/dist/interfaces/cli/commands.js +101 -0
- package/dist/interfaces/cli/main.d.ts +2 -0
- package/dist/interfaces/cli/main.js +3 -0
- package/dist/interfaces/http/cli-options.d.ts +9 -0
- package/dist/interfaces/http/cli-options.js +81 -0
- package/dist/interfaces/http/server.d.ts +14 -0
- package/dist/interfaces/http/server.js +169 -0
- package/dist/interfaces/mcp/cli-options.d.ts +6 -0
- package/dist/interfaces/mcp/cli-options.js +36 -0
- package/dist/interfaces/mcp/main.d.ts +2 -0
- package/dist/interfaces/mcp/main.js +59 -0
- package/dist/interfaces/mcp/server.d.ts +8 -0
- package/dist/interfaces/mcp/server.js +521 -0
- package/dist/runtime/index.d.ts +29 -0
- package/dist/runtime/index.js +19 -0
- package/docs/architecture.md +100 -0
- package/docs/assets/image.png +0 -0
- package/docs/mcp-activity-retrieval.md +377 -0
- package/package.json +81 -6
- package/pyproject.toml +28 -0
- package/python/__init__.py +1 -0
- package/python/catence/__init__.py +0 -0
- package/python/catence/providers/__init__.py +0 -0
- package/python/catence/providers/garmin/__init__.py +3 -0
- package/python/catence/providers/garmin/cli.py +62 -0
- package/python/catence/providers/garmin/contracts.py +23 -0
- package/python/catence/providers/garmin/registry.py +48 -0
- package/python/catence/providers/garmin/staging.py +87 -0
- package/python/catence/providers/garmin/streams.py +195 -0
- package/python/catence/providers/garmin/worker.py +450 -0
- package/python/catence/providers/strava/__init__.py +3 -0
- package/python/catence/providers/strava/cli.py +404 -0
- package/python/catence/providers/strava/staging.py +73 -0
- package/uv.lock +708 -0
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export type Migration = {
|
|
2
|
+
version: number;
|
|
3
|
+
name: string;
|
|
4
|
+
sql: string;
|
|
5
|
+
/**
|
|
6
|
+
* Idempotent statements that must run after the migration transaction
|
|
7
|
+
* commits. This is reserved for engine operations that cannot safely be
|
|
8
|
+
* combined with a populated-table rewrite on all supported DuckDB builds.
|
|
9
|
+
*/
|
|
10
|
+
postCommitSql?: string;
|
|
11
|
+
};
|
|
12
|
+
export declare const migrations: Migration[];
|
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
export const migrations = [
|
|
2
|
+
{
|
|
3
|
+
version: 1,
|
|
4
|
+
name: 'initial_ingestion_schema',
|
|
5
|
+
sql: `
|
|
6
|
+
CREATE TABLE IF NOT EXISTS schema_migrations (
|
|
7
|
+
version INTEGER PRIMARY KEY,
|
|
8
|
+
name VARCHAR NOT NULL,
|
|
9
|
+
applied_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
10
|
+
);
|
|
11
|
+
CREATE TABLE IF NOT EXISTS source_accounts (
|
|
12
|
+
provider VARCHAR NOT NULL,
|
|
13
|
+
remote_account_id VARCHAR NOT NULL,
|
|
14
|
+
display_name VARCHAR,
|
|
15
|
+
fetched_at TIMESTAMPTZ NOT NULL,
|
|
16
|
+
payload_json JSON NOT NULL,
|
|
17
|
+
PRIMARY KEY (provider, remote_account_id)
|
|
18
|
+
);
|
|
19
|
+
CREATE TABLE IF NOT EXISTS sync_runs (
|
|
20
|
+
run_id VARCHAR PRIMARY KEY,
|
|
21
|
+
provider VARCHAR NOT NULL,
|
|
22
|
+
from_date DATE NOT NULL,
|
|
23
|
+
started_at TIMESTAMPTZ NOT NULL,
|
|
24
|
+
completed_at TIMESTAMPTZ,
|
|
25
|
+
status VARCHAR NOT NULL,
|
|
26
|
+
parser_version INTEGER NOT NULL,
|
|
27
|
+
error_count INTEGER NOT NULL DEFAULT 0
|
|
28
|
+
);
|
|
29
|
+
CREATE TABLE IF NOT EXISTS sync_items (
|
|
30
|
+
item_id VARCHAR PRIMARY KEY,
|
|
31
|
+
run_id VARCHAR NOT NULL,
|
|
32
|
+
provider VARCHAR NOT NULL,
|
|
33
|
+
endpoint VARCHAR NOT NULL,
|
|
34
|
+
remote_id VARCHAR,
|
|
35
|
+
scope_json JSON NOT NULL,
|
|
36
|
+
status VARCHAR NOT NULL,
|
|
37
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
38
|
+
error_message VARCHAR,
|
|
39
|
+
started_at TIMESTAMPTZ,
|
|
40
|
+
completed_at TIMESTAMPTZ
|
|
41
|
+
);
|
|
42
|
+
CREATE TABLE IF NOT EXISTS raw_objects (
|
|
43
|
+
content_hash VARCHAR PRIMARY KEY,
|
|
44
|
+
provider VARCHAR NOT NULL,
|
|
45
|
+
endpoint VARCHAR NOT NULL,
|
|
46
|
+
remote_id VARCHAR,
|
|
47
|
+
fetched_at TIMESTAMPTZ NOT NULL,
|
|
48
|
+
content_type VARCHAR NOT NULL,
|
|
49
|
+
relative_path VARCHAR NOT NULL,
|
|
50
|
+
scope_json JSON NOT NULL,
|
|
51
|
+
parser_version INTEGER NOT NULL
|
|
52
|
+
);
|
|
53
|
+
CREATE TABLE IF NOT EXISTS source_entities (
|
|
54
|
+
provider VARCHAR NOT NULL,
|
|
55
|
+
entity_type VARCHAR NOT NULL,
|
|
56
|
+
remote_id VARCHAR NOT NULL,
|
|
57
|
+
parent_remote_id VARCHAR,
|
|
58
|
+
occurred_on DATE,
|
|
59
|
+
source_updated_at TIMESTAMPTZ,
|
|
60
|
+
raw_object_hash VARCHAR,
|
|
61
|
+
payload_json JSON NOT NULL,
|
|
62
|
+
extension_json JSON NOT NULL,
|
|
63
|
+
normalized_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
64
|
+
PRIMARY KEY (provider, entity_type, remote_id)
|
|
65
|
+
);
|
|
66
|
+
CREATE TABLE IF NOT EXISTS normalization_errors (
|
|
67
|
+
error_id VARCHAR PRIMARY KEY,
|
|
68
|
+
run_id VARCHAR NOT NULL,
|
|
69
|
+
provider VARCHAR NOT NULL,
|
|
70
|
+
endpoint VARCHAR NOT NULL,
|
|
71
|
+
remote_id VARCHAR,
|
|
72
|
+
message VARCHAR NOT NULL,
|
|
73
|
+
retryable BOOLEAN NOT NULL,
|
|
74
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
75
|
+
resolved_at TIMESTAMPTZ
|
|
76
|
+
);
|
|
77
|
+
CREATE TABLE IF NOT EXISTS activities (
|
|
78
|
+
activity_id VARCHAR PRIMARY KEY,
|
|
79
|
+
started_at_utc TIMESTAMPTZ,
|
|
80
|
+
started_at_local VARCHAR,
|
|
81
|
+
timezone VARCHAR,
|
|
82
|
+
sport VARCHAR,
|
|
83
|
+
name VARCHAR,
|
|
84
|
+
link_state VARCHAR NOT NULL
|
|
85
|
+
);
|
|
86
|
+
CREATE TABLE IF NOT EXISTS activity_sources (
|
|
87
|
+
activity_source_id VARCHAR PRIMARY KEY,
|
|
88
|
+
activity_id VARCHAR NOT NULL,
|
|
89
|
+
provider VARCHAR NOT NULL,
|
|
90
|
+
remote_activity_id VARCHAR NOT NULL,
|
|
91
|
+
external_id VARCHAR,
|
|
92
|
+
raw_object_hash VARCHAR,
|
|
93
|
+
UNIQUE (provider, remote_activity_id)
|
|
94
|
+
);
|
|
95
|
+
CREATE TABLE IF NOT EXISTS activity_summaries (
|
|
96
|
+
activity_source_id VARCHAR PRIMARY KEY,
|
|
97
|
+
distance_m DOUBLE,
|
|
98
|
+
moving_s DOUBLE,
|
|
99
|
+
elapsed_s DOUBLE,
|
|
100
|
+
elevation_gain_m DOUBLE,
|
|
101
|
+
calories DOUBLE,
|
|
102
|
+
avg_hr DOUBLE,
|
|
103
|
+
max_hr DOUBLE,
|
|
104
|
+
avg_power DOUBLE,
|
|
105
|
+
weighted_power DOUBLE,
|
|
106
|
+
avg_cadence DOUBLE,
|
|
107
|
+
training_load DOUBLE,
|
|
108
|
+
rpe DOUBLE,
|
|
109
|
+
feel DOUBLE,
|
|
110
|
+
metrics_json JSON NOT NULL
|
|
111
|
+
);
|
|
112
|
+
CREATE TABLE IF NOT EXISTS activity_intervals (
|
|
113
|
+
activity_source_id VARCHAR NOT NULL,
|
|
114
|
+
interval_key VARCHAR NOT NULL,
|
|
115
|
+
label VARCHAR,
|
|
116
|
+
start_s DOUBLE,
|
|
117
|
+
end_s DOUBLE,
|
|
118
|
+
distance_m DOUBLE,
|
|
119
|
+
avg_power DOUBLE,
|
|
120
|
+
avg_hr DOUBLE,
|
|
121
|
+
avg_pace DOUBLE,
|
|
122
|
+
intensity DOUBLE,
|
|
123
|
+
metrics_json JSON NOT NULL,
|
|
124
|
+
PRIMARY KEY (activity_source_id, interval_key)
|
|
125
|
+
);
|
|
126
|
+
CREATE TABLE IF NOT EXISTS daily_metrics (
|
|
127
|
+
provider VARCHAR NOT NULL,
|
|
128
|
+
metric_date DATE NOT NULL,
|
|
129
|
+
metric_name VARCHAR NOT NULL,
|
|
130
|
+
value_number DOUBLE,
|
|
131
|
+
value_text VARCHAR,
|
|
132
|
+
unit VARCHAR,
|
|
133
|
+
raw_object_hash VARCHAR,
|
|
134
|
+
PRIMARY KEY (provider, metric_date, metric_name)
|
|
135
|
+
);
|
|
136
|
+
CREATE TABLE IF NOT EXISTS nutrition_days (
|
|
137
|
+
provider VARCHAR NOT NULL,
|
|
138
|
+
nutrition_date DATE NOT NULL,
|
|
139
|
+
energy_kcal DOUBLE,
|
|
140
|
+
carbohydrates_g DOUBLE,
|
|
141
|
+
protein_g DOUBLE,
|
|
142
|
+
fat_g DOUBLE,
|
|
143
|
+
hydration_ml DOUBLE,
|
|
144
|
+
metrics_json JSON NOT NULL,
|
|
145
|
+
raw_object_hash VARCHAR,
|
|
146
|
+
PRIMARY KEY (provider, nutrition_date)
|
|
147
|
+
);
|
|
148
|
+
CREATE TABLE IF NOT EXISTS nutrition_items (
|
|
149
|
+
provider VARCHAR NOT NULL,
|
|
150
|
+
remote_item_id VARCHAR NOT NULL,
|
|
151
|
+
nutrition_date DATE NOT NULL,
|
|
152
|
+
meal VARCHAR,
|
|
153
|
+
consumed_at TIMESTAMPTZ,
|
|
154
|
+
food_name VARCHAR,
|
|
155
|
+
quantity DOUBLE,
|
|
156
|
+
energy_kcal DOUBLE,
|
|
157
|
+
carbohydrates_g DOUBLE,
|
|
158
|
+
protein_g DOUBLE,
|
|
159
|
+
fat_g DOUBLE,
|
|
160
|
+
payload_json JSON NOT NULL,
|
|
161
|
+
raw_object_hash VARCHAR,
|
|
162
|
+
PRIMARY KEY (provider, remote_item_id)
|
|
163
|
+
);
|
|
164
|
+
CREATE TABLE IF NOT EXISTS domain_entities (
|
|
165
|
+
provider VARCHAR NOT NULL,
|
|
166
|
+
entity_type VARCHAR NOT NULL,
|
|
167
|
+
remote_id VARCHAR NOT NULL,
|
|
168
|
+
parent_remote_id VARCHAR,
|
|
169
|
+
occurred_on DATE,
|
|
170
|
+
payload_json JSON NOT NULL,
|
|
171
|
+
extension_json JSON NOT NULL,
|
|
172
|
+
raw_object_hash VARCHAR,
|
|
173
|
+
PRIMARY KEY (provider, entity_type, remote_id)
|
|
174
|
+
);
|
|
175
|
+
CREATE TABLE IF NOT EXISTS stream_manifest (
|
|
176
|
+
provider VARCHAR NOT NULL,
|
|
177
|
+
activity_source_id VARCHAR NOT NULL,
|
|
178
|
+
content_hash VARCHAR NOT NULL,
|
|
179
|
+
relative_path VARCHAR NOT NULL,
|
|
180
|
+
row_count BIGINT NOT NULL,
|
|
181
|
+
start_at TIMESTAMPTZ,
|
|
182
|
+
end_at TIMESTAMPTZ,
|
|
183
|
+
columns_json JSON NOT NULL,
|
|
184
|
+
raw_object_hash VARCHAR,
|
|
185
|
+
PRIMARY KEY (provider, activity_source_id, content_hash)
|
|
186
|
+
);
|
|
187
|
+
CREATE OR REPLACE VIEW daily_health AS
|
|
188
|
+
SELECT provider, metric_date,
|
|
189
|
+
max(value_number) FILTER (WHERE metric_name = 'resting_hr_bpm') AS resting_hr_bpm,
|
|
190
|
+
max(value_number) FILTER (WHERE metric_name = 'hrv_ms') AS hrv_ms,
|
|
191
|
+
max(value_number) FILTER (WHERE metric_name = 'sleep_seconds') AS sleep_seconds,
|
|
192
|
+
max(value_number) FILTER (WHERE metric_name = 'sleep_score') AS sleep_score,
|
|
193
|
+
max(value_number) FILTER (WHERE metric_name = 'stress') AS stress,
|
|
194
|
+
max(value_number) FILTER (WHERE metric_name = 'body_battery') AS body_battery,
|
|
195
|
+
max(value_number) FILTER (WHERE metric_name = 'readiness') AS readiness,
|
|
196
|
+
max(value_number) FILTER (WHERE metric_name = 'spo2_pct') AS spo2_pct,
|
|
197
|
+
max(value_number) FILTER (WHERE metric_name = 'weight_kg') AS weight_kg,
|
|
198
|
+
max(value_number) FILTER (WHERE metric_name = 'steps') AS steps
|
|
199
|
+
FROM daily_metrics
|
|
200
|
+
GROUP BY provider, metric_date;
|
|
201
|
+
CREATE OR REPLACE VIEW canonical_activity_training AS
|
|
202
|
+
SELECT * EXCLUDE (priority, row_rank)
|
|
203
|
+
FROM (
|
|
204
|
+
SELECT a.*, s.provider, summary.*, CASE WHEN s.provider = 'intervals' THEN 0 ELSE 1 END AS priority,
|
|
205
|
+
row_number() OVER (PARTITION BY a.activity_id ORDER BY CASE WHEN s.provider = 'intervals' THEN 0 ELSE 1 END) AS row_rank
|
|
206
|
+
FROM activities a
|
|
207
|
+
JOIN activity_sources s USING (activity_id)
|
|
208
|
+
LEFT JOIN activity_summaries summary USING (activity_source_id)
|
|
209
|
+
)
|
|
210
|
+
WHERE row_rank = 1;
|
|
211
|
+
`,
|
|
212
|
+
},
|
|
213
|
+
{
|
|
214
|
+
version: 2,
|
|
215
|
+
name: 'domain_projection_views',
|
|
216
|
+
sql: `
|
|
217
|
+
CREATE OR REPLACE VIEW events AS
|
|
218
|
+
SELECT provider, remote_id AS event_id, occurred_on, parent_remote_id, payload_json, extension_json, raw_object_hash
|
|
219
|
+
FROM domain_entities WHERE entity_type = 'event';
|
|
220
|
+
CREATE OR REPLACE VIEW workouts AS
|
|
221
|
+
SELECT provider, remote_id AS workout_id, occurred_on, parent_remote_id, payload_json, extension_json, raw_object_hash
|
|
222
|
+
FROM domain_entities WHERE entity_type IN ('workout', 'scheduled_workout');
|
|
223
|
+
CREATE OR REPLACE VIEW workout_documents AS
|
|
224
|
+
SELECT provider, remote_id AS workout_id, payload_json->'workout_doc' AS workout_doc, payload_json
|
|
225
|
+
FROM domain_entities WHERE entity_type IN ('workout', 'scheduled_workout');
|
|
226
|
+
CREATE OR REPLACE VIEW training_plans AS
|
|
227
|
+
SELECT provider, remote_id AS training_plan_id, occurred_on, payload_json, extension_json, raw_object_hash
|
|
228
|
+
FROM domain_entities WHERE entity_type = 'training_plan';
|
|
229
|
+
CREATE OR REPLACE VIEW routes AS
|
|
230
|
+
SELECT provider, remote_id AS route_id, payload_json, extension_json, raw_object_hash
|
|
231
|
+
FROM domain_entities WHERE entity_type = 'route';
|
|
232
|
+
CREATE OR REPLACE VIEW gear AS
|
|
233
|
+
SELECT provider, remote_id AS gear_id, payload_json, extension_json, raw_object_hash
|
|
234
|
+
FROM domain_entities WHERE entity_type = 'gear';
|
|
235
|
+
CREATE OR REPLACE VIEW devices AS
|
|
236
|
+
SELECT provider, remote_id AS device_id, payload_json, extension_json, raw_object_hash
|
|
237
|
+
FROM domain_entities WHERE entity_type = 'device';
|
|
238
|
+
CREATE OR REPLACE VIEW goals AS
|
|
239
|
+
SELECT provider, remote_id AS goal_id, payload_json, extension_json, raw_object_hash
|
|
240
|
+
FROM domain_entities WHERE entity_type IN ('goal', 'challenge');
|
|
241
|
+
CREATE OR REPLACE VIEW achievements AS
|
|
242
|
+
SELECT provider, remote_id AS achievement_id, payload_json, extension_json, raw_object_hash
|
|
243
|
+
FROM domain_entities WHERE entity_type IN ('badge', 'personal_record');
|
|
244
|
+
CREATE OR REPLACE VIEW messages AS
|
|
245
|
+
SELECT provider, remote_id AS message_id, parent_remote_id, occurred_on, payload_json, extension_json, raw_object_hash
|
|
246
|
+
FROM domain_entities WHERE entity_type = 'message';
|
|
247
|
+
`,
|
|
248
|
+
},
|
|
249
|
+
{
|
|
250
|
+
version: 3,
|
|
251
|
+
name: 'retrieval_documents',
|
|
252
|
+
sql: `
|
|
253
|
+
CREATE TABLE IF NOT EXISTS retrieval_documents (
|
|
254
|
+
document_id VARCHAR PRIMARY KEY,
|
|
255
|
+
entity_type VARCHAR NOT NULL,
|
|
256
|
+
entity_id VARCHAR NOT NULL,
|
|
257
|
+
activity_source_id VARCHAR,
|
|
258
|
+
provider VARCHAR,
|
|
259
|
+
occurred_on DATE,
|
|
260
|
+
document_text VARCHAR NOT NULL,
|
|
261
|
+
metadata_json JSON NOT NULL,
|
|
262
|
+
content_hash VARCHAR NOT NULL UNIQUE,
|
|
263
|
+
generated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
|
264
|
+
);
|
|
265
|
+
CREATE INDEX IF NOT EXISTS retrieval_documents_entity_idx
|
|
266
|
+
ON retrieval_documents (entity_type, entity_id);
|
|
267
|
+
CREATE INDEX IF NOT EXISTS retrieval_documents_date_idx
|
|
268
|
+
ON retrieval_documents (occurred_on);
|
|
269
|
+
CREATE TABLE IF NOT EXISTS retrieval_index_state (
|
|
270
|
+
index_name VARCHAR PRIMARY KEY,
|
|
271
|
+
status VARCHAR NOT NULL,
|
|
272
|
+
mode VARCHAR NOT NULL,
|
|
273
|
+
source_watermark VARCHAR,
|
|
274
|
+
built_at TIMESTAMPTZ,
|
|
275
|
+
detail_json JSON NOT NULL DEFAULT '{}'
|
|
276
|
+
);
|
|
277
|
+
INSERT INTO retrieval_index_state (index_name, status, mode, detail_json)
|
|
278
|
+
VALUES ('context', 'stale', 'keyword', '{}')
|
|
279
|
+
ON CONFLICT (index_name) DO NOTHING;
|
|
280
|
+
`,
|
|
281
|
+
},
|
|
282
|
+
{
|
|
283
|
+
version: 4,
|
|
284
|
+
name: 'query_projection_views',
|
|
285
|
+
sql: `
|
|
286
|
+
CREATE OR REPLACE VIEW activity_summary_facts AS
|
|
287
|
+
SELECT summary.*, source.activity_id, source.provider, source.remote_activity_id,
|
|
288
|
+
source.raw_object_hash, activity.started_at_utc, activity.started_at_local,
|
|
289
|
+
activity.timezone, activity.sport, activity.name
|
|
290
|
+
FROM activity_summaries summary
|
|
291
|
+
JOIN activity_sources source USING (activity_source_id)
|
|
292
|
+
JOIN activities activity USING (activity_id);
|
|
293
|
+
CREATE OR REPLACE VIEW activity_interval_facts AS
|
|
294
|
+
SELECT intervals.*, source.activity_id, source.provider,
|
|
295
|
+
activity.started_at_utc, activity.sport, activity.name
|
|
296
|
+
FROM activity_intervals intervals
|
|
297
|
+
JOIN activity_sources source USING (activity_source_id)
|
|
298
|
+
JOIN activities activity USING (activity_id);
|
|
299
|
+
`,
|
|
300
|
+
},
|
|
301
|
+
{
|
|
302
|
+
version: 5,
|
|
303
|
+
name: 'incremental_sync_cursors',
|
|
304
|
+
sql: `
|
|
305
|
+
CREATE TABLE IF NOT EXISTS sync_cursors (
|
|
306
|
+
provider VARCHAR NOT NULL,
|
|
307
|
+
cursor_name VARCHAR NOT NULL,
|
|
308
|
+
covered_through_date DATE NOT NULL,
|
|
309
|
+
latest_source_date DATE,
|
|
310
|
+
lookback_days INTEGER NOT NULL,
|
|
311
|
+
last_successful_run_id VARCHAR,
|
|
312
|
+
last_completed_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
313
|
+
status VARCHAR NOT NULL,
|
|
314
|
+
detail_json JSON NOT NULL DEFAULT '{}',
|
|
315
|
+
PRIMARY KEY (provider, cursor_name)
|
|
316
|
+
);
|
|
317
|
+
CREATE TABLE IF NOT EXISTS activity_sync_state (
|
|
318
|
+
provider VARCHAR NOT NULL,
|
|
319
|
+
remote_activity_id VARCHAR NOT NULL,
|
|
320
|
+
summary_hash VARCHAR NOT NULL,
|
|
321
|
+
last_seen_run_id VARCHAR NOT NULL,
|
|
322
|
+
last_seen_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
323
|
+
last_detail_sync_at TIMESTAMPTZ,
|
|
324
|
+
details_status VARCHAR NOT NULL,
|
|
325
|
+
PRIMARY KEY (provider, remote_activity_id)
|
|
326
|
+
);
|
|
327
|
+
CREATE INDEX IF NOT EXISTS activity_sync_state_hash_idx
|
|
328
|
+
ON activity_sync_state (provider, summary_hash);
|
|
329
|
+
`,
|
|
330
|
+
},
|
|
331
|
+
];
|
|
332
|
+
//# sourceMappingURL=migrations.legacy.js.map
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { CatencePaths } from '../../contracts/runtime.js';
|
|
2
|
+
export declare class DataWriteBusyError extends Error {
|
|
3
|
+
constructor();
|
|
4
|
+
}
|
|
5
|
+
/** Serializes short writer transactions across local CLI and stdio MCP processes. */
|
|
6
|
+
export declare function withDataWriteLock<T>(paths: CatencePaths, work: () => Promise<T>): Promise<T>;
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { open, unlink } from 'node:fs/promises';
|
|
2
|
+
export class DataWriteBusyError extends Error {
|
|
3
|
+
constructor() {
|
|
4
|
+
super('Another Catence sync or enrichment request currently owns the data directory. Retry after it completes.');
|
|
5
|
+
this.name = 'DataWriteBusyError';
|
|
6
|
+
}
|
|
7
|
+
}
|
|
8
|
+
/** Serializes short writer transactions across local CLI and stdio MCP processes. */
|
|
9
|
+
export async function withDataWriteLock(paths, work) {
|
|
10
|
+
let handle;
|
|
11
|
+
try {
|
|
12
|
+
handle = await open(paths.lock, 'wx', 0o600);
|
|
13
|
+
await handle.writeFile(`${JSON.stringify({ pid: process.pid, startedAt: new Date().toISOString() })}\n`);
|
|
14
|
+
}
|
|
15
|
+
catch (error) {
|
|
16
|
+
if (error.code === 'EEXIST')
|
|
17
|
+
throw new DataWriteBusyError();
|
|
18
|
+
throw error;
|
|
19
|
+
}
|
|
20
|
+
try {
|
|
21
|
+
return await work();
|
|
22
|
+
}
|
|
23
|
+
finally {
|
|
24
|
+
await handle.close().catch(() => undefined);
|
|
25
|
+
await unlink(paths.lock).catch(() => undefined);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=write-lock.js.map
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { CatencePaths } from '../contracts/runtime.js';
|
|
2
|
+
import type { CatenceDatabase } from './storage/database.js';
|
|
3
|
+
export type ActivitySample = {
|
|
4
|
+
activity_source_id: string;
|
|
5
|
+
timestamp_utc: string | null;
|
|
6
|
+
elapsed_s: number | null;
|
|
7
|
+
distance_m: number | null;
|
|
8
|
+
latitude: number | null;
|
|
9
|
+
longitude: number | null;
|
|
10
|
+
altitude_m: number | null;
|
|
11
|
+
heart_rate_bpm: number | null;
|
|
12
|
+
power_w: number | null;
|
|
13
|
+
cadence_rpm: number | null;
|
|
14
|
+
speed_mps: number | null;
|
|
15
|
+
temperature_c: number | null;
|
|
16
|
+
grade_pct: number | null;
|
|
17
|
+
extras_json: string;
|
|
18
|
+
};
|
|
19
|
+
export declare function intervalsStreamsToSamples(activitySourceId: string, streams: Array<Record<string, unknown>>, startAt: string | null): ActivitySample[];
|
|
20
|
+
export declare function writeParquetSamples(database: CatenceDatabase, paths: CatencePaths, provider: 'intervals' | 'garmin', activityRemoteId: string, startDate: string, samples: ActivitySample[]): Promise<{
|
|
21
|
+
relativePath: string;
|
|
22
|
+
contentHash: string;
|
|
23
|
+
columns: string[];
|
|
24
|
+
}>;
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { mkdir, rename, writeFile } from 'node:fs/promises';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import { sqlString } from './storage/sql.js';
|
|
5
|
+
const knownColumns = {
|
|
6
|
+
time: 'elapsed_s', elapsed_time: 'elapsed_s', elapsed_s: 'elapsed_s', distance: 'distance_m', lat: 'latitude', latitude: 'latitude',
|
|
7
|
+
lng: 'longitude', lon: 'longitude', longitude: 'longitude', altitude: 'altitude_m', elevation: 'altitude_m', heartrate: 'heart_rate_bpm',
|
|
8
|
+
heart_rate: 'heart_rate_bpm', hr: 'heart_rate_bpm', watts: 'power_w', power: 'power_w', cadence: 'cadence_rpm', velocity_smooth: 'speed_mps',
|
|
9
|
+
speed: 'speed_mps', temp: 'temperature_c', temperature: 'temperature_c', grade_smooth: 'grade_pct', grade: 'grade_pct',
|
|
10
|
+
};
|
|
11
|
+
function numeric(value) {
|
|
12
|
+
return typeof value === 'number' && Number.isFinite(value) ? value : null;
|
|
13
|
+
}
|
|
14
|
+
export function intervalsStreamsToSamples(activitySourceId, streams, startAt) {
|
|
15
|
+
const arrays = new Map();
|
|
16
|
+
for (const stream of streams) {
|
|
17
|
+
const name = typeof stream.type === 'string' ? stream.type : typeof stream.name === 'string' ? stream.name : null;
|
|
18
|
+
const values = Array.isArray(stream.data) ? stream.data : null;
|
|
19
|
+
if (name && values)
|
|
20
|
+
arrays.set(name, values);
|
|
21
|
+
}
|
|
22
|
+
const count = Math.max(0, ...[...arrays.values()].map((values) => values.length));
|
|
23
|
+
const origin = startAt ? Date.parse(startAt) : Number.NaN;
|
|
24
|
+
return Array.from({ length: count }, (_, index) => {
|
|
25
|
+
const extras = {};
|
|
26
|
+
const sample = {
|
|
27
|
+
activity_source_id: activitySourceId, timestamp_utc: Number.isFinite(origin) ? new Date(origin + (numeric(arrays.get('time')?.[index]) ?? index) * 1000).toISOString() : null,
|
|
28
|
+
elapsed_s: null, distance_m: null, latitude: null, longitude: null, altitude_m: null, heart_rate_bpm: null, power_w: null,
|
|
29
|
+
cadence_rpm: null, speed_mps: null, temperature_c: null, grade_pct: null, extras_json: '{}',
|
|
30
|
+
};
|
|
31
|
+
for (const [name, values] of arrays) {
|
|
32
|
+
const value = values[index];
|
|
33
|
+
const column = knownColumns[name];
|
|
34
|
+
if (column)
|
|
35
|
+
sample[column] = numeric(value);
|
|
36
|
+
else
|
|
37
|
+
extras[name] = value;
|
|
38
|
+
}
|
|
39
|
+
sample.extras_json = JSON.stringify(extras);
|
|
40
|
+
return sample;
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
export async function writeParquetSamples(database, paths, provider, activityRemoteId, startDate, samples) {
|
|
44
|
+
const activitySourceId = `${provider}:${activityRemoteId}`;
|
|
45
|
+
const year = startDate.slice(0, 4) || 'unknown';
|
|
46
|
+
const month = startDate.slice(5, 7) || 'unknown';
|
|
47
|
+
const directory = path.join(paths.lake, `provider=${provider}`, `year=${year}`, `month=${month}`);
|
|
48
|
+
await mkdir(directory, { recursive: true });
|
|
49
|
+
const jsonPath = path.join(paths.staging, `${provider}-${activityRemoteId}-${process.pid}.ndjson`);
|
|
50
|
+
await writeFile(jsonPath, samples.map((sample) => JSON.stringify(sample)).join('\n'));
|
|
51
|
+
const target = path.join(directory, `activity=${activityRemoteId}.parquet`);
|
|
52
|
+
const temporary = `${target}.${process.pid}.tmp`;
|
|
53
|
+
await database.run(`COPY (SELECT * FROM read_ndjson_auto(${sqlString(jsonPath)})) TO ${sqlString(temporary)} (FORMAT PARQUET, COMPRESSION ZSTD)`);
|
|
54
|
+
await rename(temporary, target);
|
|
55
|
+
const contentHash = createHash('sha256').update(await (await import('node:fs/promises')).readFile(target)).digest('hex');
|
|
56
|
+
return {
|
|
57
|
+
relativePath: path.relative(paths.root, target),
|
|
58
|
+
contentHash,
|
|
59
|
+
columns: ['activity_source_id', 'timestamp_utc', 'elapsed_s', 'distance_m', 'latitude', 'longitude', 'altitude_m', 'heart_rate_bpm', 'power_w', 'cadence_rpm', 'speed_mps', 'temperature_c', 'grade_pct', 'extras_json'],
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
//# sourceMappingURL=streams.js.map
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { Command } from 'commander';
|
|
3
|
+
import { connectStrava, connectStravaWithCallback, createDemoStore, dataStatus, disconnectStravaAccount, initializeDataStore, linkActivity, rebuildRetrievalIndex, resolvePaths, retryDataSync, syncData, unlinkActivity } from '../../runtime/index.js';
|
|
4
|
+
const program = new Command()
|
|
5
|
+
.name('catence-data')
|
|
6
|
+
.description('Local fitness data extraction, targeted enrichment, and normalization.')
|
|
7
|
+
.option('--data-dir <directory>', 'directory holding the local Catence database and artifacts', process.env.CATENCE_DATA_DIR ?? '.catence');
|
|
8
|
+
function currentPaths() {
|
|
9
|
+
return resolvePaths(program.opts().dataDir);
|
|
10
|
+
}
|
|
11
|
+
function demoPaths() {
|
|
12
|
+
const dataDirWasProvided = process.argv.some((argument) => argument === '--data-dir' || argument.startsWith('--data-dir='));
|
|
13
|
+
const dataDir = dataDirWasProvided || process.env.CATENCE_DATA_DIR
|
|
14
|
+
? program.opts().dataDir
|
|
15
|
+
: './catence-demo';
|
|
16
|
+
return resolvePaths(dataDir);
|
|
17
|
+
}
|
|
18
|
+
program.command('init')
|
|
19
|
+
.description('Create an empty local data store without contacting a provider.')
|
|
20
|
+
.action(async () => {
|
|
21
|
+
process.stdout.write(`${JSON.stringify(await initializeDataStore(currentPaths()), null, 2)}\n`);
|
|
22
|
+
});
|
|
23
|
+
program.command('demo')
|
|
24
|
+
.description('Create a generated local demo store without contacting providers. Refuses to overwrite a non-demo data directory.')
|
|
25
|
+
.option('--days <days>', 'generated days of wellness and training data', '90')
|
|
26
|
+
.option('--seed <seed>', 'deterministic generator seed', '17')
|
|
27
|
+
.action(async (options) => {
|
|
28
|
+
const days = Number(options.days);
|
|
29
|
+
const seed = Number(options.seed);
|
|
30
|
+
process.stdout.write(`${JSON.stringify(await createDemoStore(demoPaths(), { days, seed }), null, 2)}\n`);
|
|
31
|
+
});
|
|
32
|
+
program.command('sync')
|
|
33
|
+
.requiredOption('--provider <provider>', 'intervals, garmin, strava, or all')
|
|
34
|
+
.option('--from <date>', 'explicit ISO date range; otherwise use incremental cursors')
|
|
35
|
+
.option('--refresh', 're-fetch Garmin activity details, files, and streams even when summaries are unchanged')
|
|
36
|
+
.action(async (options) => {
|
|
37
|
+
if (!['intervals', 'garmin', 'strava', 'all'].includes(options.provider))
|
|
38
|
+
throw new Error('--provider must be intervals, garmin, strava, or all.');
|
|
39
|
+
process.stdout.write(`${JSON.stringify(await syncData(currentPaths(), options.provider, options.from, true, options.refresh ?? false), null, 2)}\n`);
|
|
40
|
+
});
|
|
41
|
+
program.command('backfill')
|
|
42
|
+
.requiredOption('--from <date>', 'ISO date at which to begin backfill')
|
|
43
|
+
.option('--provider <provider>', 'intervals, garmin, strava, or all', 'all')
|
|
44
|
+
.option('--refresh', 're-fetch and upsert data already covered by the requested range (including Garmin activity details, files, and streams)')
|
|
45
|
+
.action(async (options) => {
|
|
46
|
+
if (!['intervals', 'garmin', 'strava', 'all'].includes(options.provider))
|
|
47
|
+
throw new Error('--provider must be intervals, garmin, strava, or all.');
|
|
48
|
+
process.stdout.write(`${JSON.stringify(await syncData(currentPaths(), options.provider, options.from, false, options.refresh ?? false, true), null, 2)}\n`);
|
|
49
|
+
});
|
|
50
|
+
program.command('retry')
|
|
51
|
+
.requiredOption('--run <run-id>', 'failed run to retry')
|
|
52
|
+
.action(async (options) => {
|
|
53
|
+
process.stdout.write(`${JSON.stringify(await retryDataSync(currentPaths(), options.run), null, 2)}\n`);
|
|
54
|
+
});
|
|
55
|
+
program.command('auth').description('Connect a source account without storing credentials in DuckDB.')
|
|
56
|
+
.command('strava')
|
|
57
|
+
.option('--code <authorization-code>', 'OAuth authorization code returned by Strava')
|
|
58
|
+
.option('--redirect-uri <uri>', 'OAuth redirect URI; defaults depend on the selected flow')
|
|
59
|
+
.option('--callback', 'wait for a browser OAuth callback on a loopback redirect URI')
|
|
60
|
+
.option('--timeout-seconds <seconds>', 'maximum callback wait time', '300')
|
|
61
|
+
.action(async (options) => {
|
|
62
|
+
if (options.callback && options.code)
|
|
63
|
+
throw new Error('--code and --callback cannot be used together.');
|
|
64
|
+
const timeoutSeconds = Number(options.timeoutSeconds);
|
|
65
|
+
if (!Number.isInteger(timeoutSeconds) || timeoutSeconds < 1 || timeoutSeconds > 900)
|
|
66
|
+
throw new Error('--timeout-seconds must be an integer between 1 and 900.');
|
|
67
|
+
const redirectUri = options.redirectUri ?? (options.callback ? 'http://127.0.0.1:8765/strava/callback' : 'http://localhost');
|
|
68
|
+
const result = options.callback
|
|
69
|
+
? await connectStravaWithCallback(currentPaths(), redirectUri, (url) => {
|
|
70
|
+
process.stderr.write(`Open this URL in your browser to connect Strava:\n${url}\nWaiting for the local callback…\n`);
|
|
71
|
+
}, timeoutSeconds * 1_000)
|
|
72
|
+
: await connectStrava(currentPaths(), options.code, redirectUri);
|
|
73
|
+
process.stdout.write(JSON.stringify(result, null, 2) + "\n");
|
|
74
|
+
});
|
|
75
|
+
program.command('disconnect').description('Remove locally stored credentials for a source account.')
|
|
76
|
+
.command('strava')
|
|
77
|
+
.action(async () => {
|
|
78
|
+
process.stdout.write(JSON.stringify(await disconnectStravaAccount(currentPaths()), null, 2) + "\n");
|
|
79
|
+
});
|
|
80
|
+
const activityCommand = program.command('activity').description('Audit and correct logical activity links.');
|
|
81
|
+
activityCommand.command('link')
|
|
82
|
+
.requiredOption('--source <activity-source-id>')
|
|
83
|
+
.requiredOption('--activity <activity-id>')
|
|
84
|
+
.action(async (options) => {
|
|
85
|
+
process.stdout.write(JSON.stringify(await linkActivity(currentPaths(), options.source, options.activity), null, 2) + "\n");
|
|
86
|
+
});
|
|
87
|
+
activityCommand.command('unlink')
|
|
88
|
+
.requiredOption('--source <activity-source-id>')
|
|
89
|
+
.action(async (options) => {
|
|
90
|
+
process.stdout.write(JSON.stringify(await unlinkActivity(currentPaths(), options.source), null, 2) + "\n");
|
|
91
|
+
});
|
|
92
|
+
program.command('status').action(async () => {
|
|
93
|
+
process.stdout.write(`${JSON.stringify(await dataStatus(currentPaths()), null, 2)}\n`);
|
|
94
|
+
});
|
|
95
|
+
program.command('build-retrieval-index')
|
|
96
|
+
.description('Build the derived local text-retrieval index after a completed sync.')
|
|
97
|
+
.action(async () => {
|
|
98
|
+
process.stdout.write(`${JSON.stringify(await rebuildRetrievalIndex(currentPaths()), null, 2)}\n`);
|
|
99
|
+
});
|
|
100
|
+
await program.parseAsync();
|
|
101
|
+
//# sourceMappingURL=commands.js.map
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export type ServeCliOptions = {
|
|
2
|
+
dataDir?: string;
|
|
3
|
+
host: string;
|
|
4
|
+
port: number;
|
|
5
|
+
allowedOrigins: string[];
|
|
6
|
+
help: boolean;
|
|
7
|
+
};
|
|
8
|
+
export declare const SERVE_USAGE = "Usage: catence serve [options]\n\nRun Catence as a local Streamable HTTP MCP server.\n\nOptions:\n --data-dir <directory> Catence data directory. Overrides CATENCE_DATA_DIR.\n --host <host> Listener host. Defaults to 127.0.0.1.\n --port <port> Listener port. Defaults to 8787.\n --allow-origin <origin> Browser origin allowed to call Catence APIs. May be repeated.\n -h, --help Show this help text.";
|
|
9
|
+
export declare function parseServeCliOptions(arguments_: readonly string[]): ServeCliOptions;
|