catence 0.2.0-beta.12 → 0.2.0-beta.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/contracts/release.d.ts +1 -1
- package/dist/contracts/release.js +1 -1
- package/dist/elt/application/management.d.ts +13 -0
- package/dist/elt/application/management.js +78 -0
- package/dist/elt/ingestion/importer.js +1 -1
- package/dist/elt/normalization/normalizers.js +98 -9
- package/dist/elt/storage/database.d.ts +6 -0
- package/dist/elt/storage/database.js +22 -0
- package/dist/interfaces/cli/commands.js +26 -1
- package/dist/runtime/index.d.ts +1 -1
- package/dist/runtime/index.js +1 -1
- package/package.json +1 -1
- package/python/catence/providers/garmin/streams.py +38 -0
- package/python/catence/providers/garmin/swim_backfill.py +66 -0
- package/python/catence/providers/garmin/worker.py +15 -1
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* Keep these values synchronized with release/manifest.json; release checks
|
|
4
4
|
* deliberately reject drift before a package can be published.
|
|
5
5
|
*/
|
|
6
|
-
export declare const CATENCE_RUNTIME_VERSION = "0.2.0-beta.
|
|
6
|
+
export declare const CATENCE_RUNTIME_VERSION = "0.2.0-beta.13";
|
|
7
7
|
export declare const CATENCE_PROTOCOL_VERSION = 1;
|
|
8
8
|
export type CatenceRuntimeCapabilities = {
|
|
9
9
|
mcp: true;
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* Keep these values synchronized with release/manifest.json; release checks
|
|
4
4
|
* deliberately reject drift before a package can be published.
|
|
5
5
|
*/
|
|
6
|
-
export const CATENCE_RUNTIME_VERSION = '0.2.0-beta.
|
|
6
|
+
export const CATENCE_RUNTIME_VERSION = '0.2.0-beta.13';
|
|
7
7
|
export const CATENCE_PROTOCOL_VERSION = 1;
|
|
8
8
|
export function catenceRuntimeHealth() {
|
|
9
9
|
return {
|
|
@@ -24,4 +24,17 @@ export declare function unlinkActivity(paths: CatencePaths, source: string): Pro
|
|
|
24
24
|
export declare function dataStatus(paths: CatencePaths): Promise<Record<string, unknown>>;
|
|
25
25
|
export declare function syncProgress(paths: CatencePaths): Promise<SyncProgressSnapshot>;
|
|
26
26
|
export declare function rebuildRetrievalIndex(paths: CatencePaths): Promise<Record<string, unknown>>;
|
|
27
|
+
export declare function importStagedFile(paths: CatencePaths, provider: ProviderChoice, filePath: string): Promise<Record<string, unknown>>;
|
|
28
|
+
export declare function resolveExtractionErrors(paths: CatencePaths, opts?: {
|
|
29
|
+
runIds?: string[];
|
|
30
|
+
provider?: string;
|
|
31
|
+
before?: string;
|
|
32
|
+
all?: boolean;
|
|
33
|
+
}): Promise<Record<string, unknown>>;
|
|
34
|
+
/**
|
|
35
|
+
* Re-run normalization over already-captured nutrition entities without
|
|
36
|
+
* contacting any provider. Idempotent upserts let a fixed normalizer backfill
|
|
37
|
+
* previously mis-parsed nutrition_days/nutrition_items from stored payloads.
|
|
38
|
+
*/
|
|
39
|
+
export declare function reimportNutrition(paths: CatencePaths): Promise<Record<string, unknown>>;
|
|
27
40
|
export {};
|
|
@@ -14,6 +14,7 @@ import { importJsonl } from '../ingestion/importer.js';
|
|
|
14
14
|
import { IntervalsExtractor } from '../ingestion/providers/intervals.js';
|
|
15
15
|
import { completeStravaAuthorization, disconnectStrava, getStravaAuthorizationUrl, syncStravaGear } from '../ingestion/providers/strava/service.js';
|
|
16
16
|
import { setManualActivityLink, unlinkActivitySource } from '../normalization/activities/linking.js';
|
|
17
|
+
import { importSourceEntity } from '../normalization/normalizers.js';
|
|
17
18
|
import { CatenceDatabase, openReadOnlyRepository, ReadOnlyDatabaseError } from '../storage/database.js';
|
|
18
19
|
import { mergeProgress, ProgressPump, readRunningProgress, removeProgressSidecar } from '../storage/progress-sidecar.js';
|
|
19
20
|
import { withDataWriteLock } from '../storage/write-lock.js';
|
|
@@ -273,6 +274,11 @@ export async function syncData(paths, provider, explicitFrom, advanceCursor = tr
|
|
|
273
274
|
const runs = [];
|
|
274
275
|
for (const source of providers)
|
|
275
276
|
runs.push(await syncProvider(database, paths, source, { explicitFrom, toDate, advanceCursor, refreshActivities, backfill }, log));
|
|
277
|
+
const executedRuns = runs.filter((run) => !run.skipped);
|
|
278
|
+
if (executedRuns.length > 0) {
|
|
279
|
+
const index = await buildRetrievalIndex(database);
|
|
280
|
+
log.info('Rebuilt retrieval index after sync', { documents: index.documents, mode: index.mode, watermark: index.watermark });
|
|
281
|
+
}
|
|
276
282
|
return { runIds: runs.flatMap((run) => run.runId ? [run.runId] : []), runs, timedOutRuns: timedOutRuns.runIds };
|
|
277
283
|
}
|
|
278
284
|
finally {
|
|
@@ -466,4 +472,76 @@ export async function rebuildRetrievalIndex(paths) {
|
|
|
466
472
|
await database.close();
|
|
467
473
|
}
|
|
468
474
|
}
|
|
475
|
+
export async function importStagedFile(paths, provider, filePath) {
|
|
476
|
+
if (!existsSync(paths.database))
|
|
477
|
+
throw new Error('No Catence database exists yet. Complete a sync before importing staged records.');
|
|
478
|
+
if (!existsSync(filePath))
|
|
479
|
+
throw new Error(`Staging file not found: ${filePath}`);
|
|
480
|
+
const database = await CatenceDatabase.open(paths);
|
|
481
|
+
try {
|
|
482
|
+
const runId = randomUUID();
|
|
483
|
+
const importedRecords = await importJsonl(database, runId, filePath);
|
|
484
|
+
return { runId, provider, importedRecords };
|
|
485
|
+
}
|
|
486
|
+
finally {
|
|
487
|
+
await database.close();
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
export async function resolveExtractionErrors(paths, opts = {}) {
|
|
491
|
+
if (!existsSync(paths.database))
|
|
492
|
+
throw new Error('No Catence database exists yet.');
|
|
493
|
+
const database = await CatenceDatabase.open(paths);
|
|
494
|
+
try {
|
|
495
|
+
const resolved = await database.resolveErrors({
|
|
496
|
+
runIds: opts.runIds,
|
|
497
|
+
provider: opts.provider,
|
|
498
|
+
before: opts.before,
|
|
499
|
+
all: opts.all,
|
|
500
|
+
});
|
|
501
|
+
return { resolvedErrors: resolved };
|
|
502
|
+
}
|
|
503
|
+
finally {
|
|
504
|
+
await database.close();
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
/**
|
|
508
|
+
* Re-run normalization over already-captured nutrition entities without
|
|
509
|
+
* contacting any provider. Idempotent upserts let a fixed normalizer backfill
|
|
510
|
+
* previously mis-parsed nutrition_days/nutrition_items from stored payloads.
|
|
511
|
+
*/
|
|
512
|
+
export async function reimportNutrition(paths) {
|
|
513
|
+
if (!existsSync(paths.database))
|
|
514
|
+
throw new Error('No Catence database exists yet. Complete a sync before re-importing nutrition.');
|
|
515
|
+
const database = await CatenceDatabase.open(paths);
|
|
516
|
+
try {
|
|
517
|
+
const rows = await database.rows(`SELECT provider, entity_type, remote_id, parent_remote_id, occurred_on, source_updated_at, raw_object_hash,
|
|
518
|
+
cast(payload_json AS VARCHAR) AS payload_json, cast(extension_json AS VARCHAR) AS extension_json
|
|
519
|
+
FROM source_entities
|
|
520
|
+
WHERE entity_type IN ('nutrition_log', 'nutrition_day')
|
|
521
|
+
ORDER BY provider, occurred_on, remote_id`);
|
|
522
|
+
for (const row of rows) {
|
|
523
|
+
await importSourceEntity(database, {
|
|
524
|
+
kind: 'source_entity', schemaVersion: 1, provider: row.provider, entityType: row.entity_type,
|
|
525
|
+
remoteId: row.remote_id, parentRemoteId: row.parent_remote_id, occurredOn: row.occurred_on, sourceUpdatedAt: row.source_updated_at,
|
|
526
|
+
rawObjectHash: row.raw_object_hash, payload: parseJsonObject(row.payload_json), extension: parseJsonObject(row.extension_json),
|
|
527
|
+
});
|
|
528
|
+
}
|
|
529
|
+
const days = (await database.rows('SELECT count(*) AS count FROM nutrition_days'))[0]?.count ?? 0;
|
|
530
|
+
const items = (await database.rows('SELECT count(*) AS count FROM nutrition_items'))[0]?.count ?? 0;
|
|
531
|
+
return { reimportedEntities: rows.length, nutritionDays: days, nutritionItems: items };
|
|
532
|
+
}
|
|
533
|
+
finally {
|
|
534
|
+
await database.close();
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
function parseJsonObject(value) {
|
|
538
|
+
if (!value)
|
|
539
|
+
return {};
|
|
540
|
+
try {
|
|
541
|
+
return JSON.parse(value);
|
|
542
|
+
}
|
|
543
|
+
catch {
|
|
544
|
+
return {};
|
|
545
|
+
}
|
|
546
|
+
}
|
|
469
547
|
//# sourceMappingURL=management.js.map
|
|
@@ -33,7 +33,7 @@ export async function importRecord(database, runId, record, log) {
|
|
|
33
33
|
if (activity[0]?.activity_id)
|
|
34
34
|
await refreshActivityQuality(database, activity[0].activity_id);
|
|
35
35
|
}
|
|
36
|
-
if (record.provider === 'garmin' && ['activity_detail', 'activity_interval', 'activity_exercise_set'].includes(record.entityType) && record.parentRemoteId) {
|
|
36
|
+
if (record.provider === 'garmin' && ['activity_detail', 'activity_interval', 'activity_exercise_set', 'activity_swim_lengths'].includes(record.entityType) && record.parentRemoteId) {
|
|
37
37
|
const activity = await database.rows('SELECT activity_id FROM activity_sources WHERE activity_source_id = $activitySourceId', { activitySourceId: `garmin:${record.parentRemoteId}` });
|
|
38
38
|
if (activity[0]?.activity_id)
|
|
39
39
|
await refreshActivityQuality(database, activity[0].activity_id);
|
|
@@ -163,24 +163,63 @@ async function upsertDailyMetric(database, provider, metricDate, metricName, val
|
|
|
163
163
|
ON CONFLICT (provider, metric_date, metric_name) DO UPDATE SET value_number = excluded.value_number, value_text = excluded.value_text,
|
|
164
164
|
unit = excluded.unit, raw_object_hash = excluded.raw_object_hash`, { provider, metricDate, metricName, valueNumber, valueText, unit, rawHash });
|
|
165
165
|
}
|
|
166
|
+
/**
|
|
167
|
+
* Garmin's nutrition food log keeps its actuals under dailyNutritionContent
|
|
168
|
+
* (a per-day rolled-up object), while meal definitions and hydration endpoints
|
|
169
|
+
* share the same source entity type. Flat keys are kept for providers that
|
|
170
|
+
* return them directly.
|
|
171
|
+
*/
|
|
166
172
|
function nutritionFields(payload) {
|
|
173
|
+
const content = object(payload.dailyNutritionContent);
|
|
167
174
|
return {
|
|
168
|
-
energyKcal: firstNumber(payload, ['kcalConsumed', 'calories', 'totalCalories', 'energyKcal']),
|
|
169
|
-
carbohydratesG: firstNumber(payload, ['carbohydrates', 'carbs', 'totalCarbs', 'carbohydrateGrams']),
|
|
170
|
-
proteinG: firstNumber(payload, ['protein', 'totalProtein', 'proteinGrams']),
|
|
171
|
-
fatG: firstNumber(payload, ['fatTotal', 'fat', 'totalFat', 'fatGrams']),
|
|
172
|
-
hydrationMl: firstNumber(payload, ['hydrationVolume', 'hydration', 'waterMl', 'totalWater']),
|
|
175
|
+
energyKcal: firstNumber(payload, ['kcalConsumed', 'calories', 'totalCalories', 'energyKcal']) ?? firstNumber(content, ['calories', 'energyKcal', 'totalCalories']),
|
|
176
|
+
carbohydratesG: firstNumber(payload, ['carbohydrates', 'carbs', 'totalCarbs', 'carbohydrateGrams']) ?? firstNumber(content, ['carbohydrates', 'carbs']),
|
|
177
|
+
proteinG: firstNumber(payload, ['protein', 'totalProtein', 'proteinGrams']) ?? firstNumber(content, ['protein']),
|
|
178
|
+
fatG: firstNumber(payload, ['fatTotal', 'fat', 'totalFat', 'fatGrams']) ?? firstNumber(content, ['fat', 'fatTotal']),
|
|
179
|
+
hydrationMl: firstNumber(payload, ['hydrationVolume', 'hydration', 'waterMl', 'totalWater', 'valueInML', 'dailyAverageinML']),
|
|
173
180
|
};
|
|
174
181
|
}
|
|
182
|
+
/**
|
|
183
|
+
* Garmin food-log items live under mealDetails[].foods[]. Each food carries the
|
|
184
|
+
* day's nutrition_date, so a meal label is folded onto the item to avoid losing
|
|
185
|
+
* it. Flat arrays are kept for providers that return items directly.
|
|
186
|
+
*/
|
|
175
187
|
function nutritionItems(payload) {
|
|
176
|
-
const
|
|
177
|
-
|
|
188
|
+
const items = [];
|
|
189
|
+
const mealDetails = payload.mealDetails;
|
|
190
|
+
if (Array.isArray(mealDetails)) {
|
|
191
|
+
for (const detail of mealDetails) {
|
|
192
|
+
const mealDetail = object(detail);
|
|
193
|
+
const mealName = firstString(object(mealDetail.meal), ['mealName']) ?? firstString(mealDetail, ['mealName']);
|
|
194
|
+
const foods = mealDetail.foods;
|
|
195
|
+
if (Array.isArray(foods)) {
|
|
196
|
+
for (const food of foods) {
|
|
197
|
+
const item = object(food);
|
|
198
|
+
if (Object.keys(item).length === 0)
|
|
199
|
+
continue;
|
|
200
|
+
items.push(mealName ? { ...item, mealName } : item);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
else if (Object.keys(mealDetail).length > 0 && firstString(mealDetail, ['foodName', 'name', 'description']) !== null) {
|
|
204
|
+
items.push(mealName ? { ...mealDetail, mealName } : mealDetail);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
if (items.length > 0)
|
|
208
|
+
return items;
|
|
209
|
+
}
|
|
210
|
+
for (const key of ['foodItems', 'items', 'entries', 'foodLogEntries', 'meals']) {
|
|
178
211
|
const value = payload[key];
|
|
179
212
|
if (Array.isArray(value))
|
|
180
213
|
return value.map(object).filter((item) => Object.keys(item).length > 0);
|
|
181
214
|
}
|
|
182
215
|
return [];
|
|
183
216
|
}
|
|
217
|
+
/** True when a payload contributes real nutrition facts rather than meal definitions or hydration-only bookkeeping. */
|
|
218
|
+
function hasNutritionContent(payload) {
|
|
219
|
+
const fields = nutritionFields(payload);
|
|
220
|
+
return fields.energyKcal !== null || fields.carbohydratesG !== null || fields.proteinG !== null || fields.fatG !== null
|
|
221
|
+
|| fields.hydrationMl !== null || nutritionItems(payload).length > 0;
|
|
222
|
+
}
|
|
184
223
|
export async function importSourceEntity(database, entity) {
|
|
185
224
|
const payload = object(entity.payload);
|
|
186
225
|
const rawHash = entity.rawObjectHash;
|
|
@@ -223,6 +262,9 @@ export async function importSourceEntity(database, entity) {
|
|
|
223
262
|
if (entity.provider === 'garmin' && ['activity_detail', 'activity_interval', 'activity_exercise_set'].includes(entity.entityType)) {
|
|
224
263
|
await importGarminExplicitSwimLengths(database, entity.parentRemoteId, entity.entityType, payload, rawHash);
|
|
225
264
|
}
|
|
265
|
+
if (entity.provider === 'garmin' && entity.entityType === 'activity_swim_lengths') {
|
|
266
|
+
await importGarminFitSwimLengths(database, entity.parentRemoteId, payload, rawHash);
|
|
267
|
+
}
|
|
226
268
|
if (entity.entityType === 'wellness' || entity.entityType === 'daily_health')
|
|
227
269
|
await importDailyMetrics(database, entity.provider, entity.remoteId, entity.occurredOn, payload, rawHash);
|
|
228
270
|
if (entity.entityType === 'nutrition_day' || entity.entityType === 'nutrition_log')
|
|
@@ -668,6 +710,46 @@ async function importGarminExplicitSwimLengths(database, parentRemoteId, entityT
|
|
|
668
710
|
}
|
|
669
711
|
}
|
|
670
712
|
}
|
|
713
|
+
async function importGarminFitSwimLengths(database, parentRemoteId, payload, rawHash) {
|
|
714
|
+
if (!parentRemoteId)
|
|
715
|
+
return;
|
|
716
|
+
const activitySourceId = providerActivityId('garmin', parentRemoteId);
|
|
717
|
+
const lengths = objectArray(payload.lengths);
|
|
718
|
+
if (lengths.length === 0)
|
|
719
|
+
return;
|
|
720
|
+
const source = 'garmin_fit_length';
|
|
721
|
+
await database.run('DELETE FROM swim_lengths WHERE activity_source_id = $activitySourceId AND source = $source', { activitySourceId, source });
|
|
722
|
+
const summaryPoolLengthM = await activityPoolLengthM(database, activitySourceId);
|
|
723
|
+
for (const [offset, length] of lengths.entries()) {
|
|
724
|
+
const durationS = firstNumber(length, ['durationS', 'duration', 'durationSeconds', 'elapsedDuration']);
|
|
725
|
+
if (durationS === null)
|
|
726
|
+
continue;
|
|
727
|
+
const sourceLabel = swimSourceLabel(length);
|
|
728
|
+
const isRest = firstBoolean(length, ['isRest', 'rest']) ?? Boolean(sourceLabel && /rest|idle|pause/.test(sourceLabel));
|
|
729
|
+
const lengthIndex = Math.round(firstNumber(length, ['lengthIndex', 'messageIndex', 'lengthNumber', 'index']) ?? offset);
|
|
730
|
+
await database.run(`INSERT INTO swim_lengths
|
|
731
|
+
(activity_source_id, source, length_index, lap_index, pool_length_m, start_time, duration_s, active_duration_s, distance_m, stroke_count, stroke_rate, swolf, avg_hr, max_hr, is_rest, confidence, metrics_json, raw_object_hash)
|
|
732
|
+
VALUES ($activitySourceId, $source, $lengthIndex, $lapIndex, $poolLengthM, $startTime, $durationS, $activeDurationS, $distanceM, $strokeCount, $strokeRate, $swolf, $avgHr, $maxHr, $isRest, 'provider_supplied', $metrics, $rawHash)
|
|
733
|
+
ON CONFLICT (activity_source_id, source, length_index) DO UPDATE SET
|
|
734
|
+
lap_index = excluded.lap_index, pool_length_m = excluded.pool_length_m, start_time = excluded.start_time,
|
|
735
|
+
duration_s = excluded.duration_s, active_duration_s = excluded.active_duration_s, distance_m = excluded.distance_m,
|
|
736
|
+
stroke_count = excluded.stroke_count, stroke_rate = excluded.stroke_rate, swolf = excluded.swolf,
|
|
737
|
+
avg_hr = excluded.avg_hr, max_hr = excluded.max_hr, is_rest = excluded.is_rest,
|
|
738
|
+
confidence = excluded.confidence, metrics_json = excluded.metrics_json, raw_object_hash = excluded.raw_object_hash`, {
|
|
739
|
+
activitySourceId, source, lengthIndex,
|
|
740
|
+
lapIndex: firstNumber(length, ['lapIndex', 'lapNumber']), poolLengthM: garminPoolLengthM(length) ?? summaryPoolLengthM,
|
|
741
|
+
startTime: timestamp(length.startTime ?? length.startTimestamp ?? length.timestamp), durationS,
|
|
742
|
+
activeDurationS: firstNumber(length, ['activeDurationS', 'activeDuration', 'movingDuration']),
|
|
743
|
+
distanceM: firstNumber(length, ['distanceM', 'distance']),
|
|
744
|
+
strokeCount: firstNumber(length, ['strokeCount', 'strokes', 'total_strokes']),
|
|
745
|
+
strokeRate: firstNumber(length, ['strokeRate', 'averageStrokeRate', 'avg_swimming_cadence', 'avg_cadence', 'cadence']),
|
|
746
|
+
swolf: firstNumber(length, ['swolf', 'avg_swolf']),
|
|
747
|
+
avgHr: firstNumber(length, ['avgHr', 'averageHR', 'avg_heart_rate']),
|
|
748
|
+
maxHr: firstNumber(length, ['maxHr', 'maxHR', 'max_heart_rate']),
|
|
749
|
+
isRest, metrics: json(length), rawHash,
|
|
750
|
+
});
|
|
751
|
+
}
|
|
752
|
+
}
|
|
671
753
|
async function addActivityQualityFlag(database, activitySourceId, code, severity, details, rawHash) {
|
|
672
754
|
await database.run(`INSERT INTO activity_quality_flags (activity_source_id, flag_code, severity, details_json, raw_object_hash)
|
|
673
755
|
VALUES ($activitySourceId, $code, $severity, $details, $rawHash)
|
|
@@ -791,12 +873,19 @@ async function importNutrition(database, provider, date, remoteId, payload, rawH
|
|
|
791
873
|
const nutritionDate = datePart(date) ?? datePart(firstString(payload, ['id', 'date', 'calendarDate'])) ?? null;
|
|
792
874
|
if (!nutritionDate)
|
|
793
875
|
return;
|
|
876
|
+
if (!hasNutritionContent(payload))
|
|
877
|
+
return;
|
|
794
878
|
const fields = nutritionFields(payload);
|
|
795
879
|
await database.run(`INSERT INTO nutrition_days
|
|
796
880
|
(provider, nutrition_date, energy_kcal, carbohydrates_g, protein_g, fat_g, hydration_ml, metrics_json, raw_object_hash)
|
|
797
881
|
VALUES ($provider, $nutritionDate, $energyKcal, $carbohydratesG, $proteinG, $fatG, $hydrationMl, $metrics, $rawHash)
|
|
798
|
-
ON CONFLICT (provider, nutrition_date) DO UPDATE SET
|
|
799
|
-
|
|
882
|
+
ON CONFLICT (provider, nutrition_date) DO UPDATE SET
|
|
883
|
+
energy_kcal = COALESCE(excluded.energy_kcal, nutrition_days.energy_kcal),
|
|
884
|
+
carbohydrates_g = COALESCE(excluded.carbohydrates_g, nutrition_days.carbohydrates_g),
|
|
885
|
+
protein_g = COALESCE(excluded.protein_g, nutrition_days.protein_g),
|
|
886
|
+
fat_g = COALESCE(excluded.fat_g, nutrition_days.fat_g),
|
|
887
|
+
hydration_ml = COALESCE(excluded.hydration_ml, nutrition_days.hydration_ml),
|
|
888
|
+
metrics_json = excluded.metrics_json, raw_object_hash = excluded.raw_object_hash`, { provider, nutritionDate, ...fields, metrics: json(payload), rawHash });
|
|
800
889
|
for (const [index, item] of nutritionItems(payload).entries()) {
|
|
801
890
|
const itemFields = nutritionFields(item);
|
|
802
891
|
const itemId = firstString(item, ['id', 'foodId', 'entryId']) ?? `${remoteId}:${index}`;
|
|
@@ -22,6 +22,12 @@ export declare class CatenceDatabase implements WriteDataStore {
|
|
|
22
22
|
interruptStaleSyncRuns(timeoutMs?: number): Promise<InterruptedRuns>;
|
|
23
23
|
syncProgress(): Promise<SyncProgressSnapshot>;
|
|
24
24
|
addError(runId: string, provider: Provider, endpoint: string, remoteId: string | null, message: string, retryable: boolean): Promise<void>;
|
|
25
|
+
resolveErrors(opts?: {
|
|
26
|
+
runIds?: string[];
|
|
27
|
+
provider?: Provider;
|
|
28
|
+
before?: string;
|
|
29
|
+
all?: boolean;
|
|
30
|
+
}): Promise<number>;
|
|
25
31
|
status(): Promise<Record<string, unknown>>;
|
|
26
32
|
getRun(runId: string): Promise<{
|
|
27
33
|
provider: Provider;
|
|
@@ -147,6 +147,28 @@ export class CatenceDatabase {
|
|
|
147
147
|
VALUES ($errorId, $runId, $provider, $endpoint, $remoteId, $message, $retryable)`, { errorId: randomUUID(), runId, provider, endpoint, remoteId, message, retryable });
|
|
148
148
|
await this.run('UPDATE sync_runs SET error_count = error_count + 1 WHERE run_id = $runId', { runId });
|
|
149
149
|
}
|
|
150
|
+
async resolveErrors(opts = {}) {
|
|
151
|
+
const clauses = ['resolved_at IS NULL'];
|
|
152
|
+
const values = {};
|
|
153
|
+
if (opts.runIds && opts.runIds.length > 0) {
|
|
154
|
+
const placeholders = opts.runIds.map((_, index) => `$runId${index}`).join(', ');
|
|
155
|
+
clauses.push(`run_id IN (${placeholders})`);
|
|
156
|
+
opts.runIds.forEach((runId, index) => { values[`runId${index}`] = runId; });
|
|
157
|
+
}
|
|
158
|
+
if (opts.provider) {
|
|
159
|
+
clauses.push('provider = $provider');
|
|
160
|
+
values.provider = opts.provider;
|
|
161
|
+
}
|
|
162
|
+
if (opts.before) {
|
|
163
|
+
clauses.push('created_at < $before');
|
|
164
|
+
values.before = opts.before;
|
|
165
|
+
}
|
|
166
|
+
if (!opts.all && clauses.length === 1) {
|
|
167
|
+
throw new Error('Refusing to resolve all unresolved errors without an explicit --all flag.');
|
|
168
|
+
}
|
|
169
|
+
const resolved = await this.rows(`UPDATE normalization_errors SET resolved_at = now() WHERE ${clauses.join(' AND ')} RETURNING error_id`, values);
|
|
170
|
+
return resolved.length;
|
|
171
|
+
}
|
|
150
172
|
async status() {
|
|
151
173
|
// @duckdb/node-api owns native statement state on the connection. Keep
|
|
152
174
|
// status reads sequential instead of issuing several native calls at once.
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
import { createInterface } from 'node:readline/promises';
|
|
3
3
|
import { stdin as input, stdout as output } from 'node:process';
|
|
4
4
|
import { Command } from 'commander';
|
|
5
|
-
import { addAthlete, athleteStorePaths, CATENCE_RUNTIME_VERSION, channelForVersion, createDemoStore, dataStatus, defaultCatalogHome, disconnectStravaAccount, fetchNpmDistTags, fetchPypiReleases, hasCommand, initializeCatalog, isGlobalNpmInstall, linkActivity, loadCatalog, npmGlobalPrefix, planSelfUpdate, readInstalledConsoleVersion, rebuildRetrievalIndex, resolveAthlete, resolveCatalogPaths, retryDataSync, runSelfUpdateCommand, setAthleteSecret, syncData, syncProgress, connectStrava, connectStravaWithCallback, unlinkActivity, } from '../../runtime/index.js';
|
|
5
|
+
import { addAthlete, athleteStorePaths, CATENCE_RUNTIME_VERSION, channelForVersion, createDemoStore, dataStatus, defaultCatalogHome, disconnectStravaAccount, fetchNpmDistTags, fetchPypiReleases, hasCommand, importStagedFile, initializeCatalog, isGlobalNpmInstall, linkActivity, loadCatalog, npmGlobalPrefix, planSelfUpdate, readInstalledConsoleVersion, rebuildRetrievalIndex, reimportNutrition, resolveExtractionErrors, resolveAthlete, resolveCatalogPaths, retryDataSync, runSelfUpdateCommand, setAthleteSecret, syncData, syncProgress, connectStrava, connectStravaWithCallback, unlinkActivity, } from '../../runtime/index.js';
|
|
6
6
|
const program = new Command()
|
|
7
7
|
.name('catence-data')
|
|
8
8
|
.description('Athlete-scoped Catence data setup, extraction, and normalization.')
|
|
@@ -252,5 +252,30 @@ program.command('build-retrieval-index').description('Build derived local retrie
|
|
|
252
252
|
const { paths } = await currentAthlete();
|
|
253
253
|
process.stdout.write(`${JSON.stringify(await rebuildRetrievalIndex(paths), null, 2)}\n`);
|
|
254
254
|
});
|
|
255
|
+
program.command('reimport-nutrition').description('Re-run normalization over captured nutrition entities without contacting providers.').action(async () => {
|
|
256
|
+
const { paths } = await currentAthlete();
|
|
257
|
+
process.stdout.write(`${JSON.stringify(await reimportNutrition(paths), null, 2)}\n`);
|
|
258
|
+
});
|
|
259
|
+
program
|
|
260
|
+
.command('import')
|
|
261
|
+
.description('Import a staged JSONL file (for example a swim-length backfill) into one athlete’s data store.')
|
|
262
|
+
.requiredOption('--provider <provider>', 'garmin, intervals, or strava')
|
|
263
|
+
.argument('<file>', 'path to the staged JSONL file')
|
|
264
|
+
.action(async (file, options) => {
|
|
265
|
+
const { paths } = await currentAthlete();
|
|
266
|
+
process.stdout.write(`${JSON.stringify(await importStagedFile(paths, options.provider, file), null, 2)}\n`);
|
|
267
|
+
});
|
|
268
|
+
program
|
|
269
|
+
.command('resolve-errors')
|
|
270
|
+
.description('Mark unresolved extraction errors as resolved (for example stale bootstrap failures from an environment that has since recovered).')
|
|
271
|
+
.option('--run <run-id>', 'resolve errors from a specific run only (repeatable)')
|
|
272
|
+
.option('--provider <provider>', 'resolve errors for garmin, intervals, or strava only')
|
|
273
|
+
.option('--before <timestamp>', 'resolve errors created before this ISO timestamp')
|
|
274
|
+
.option('--all', 'resolve every unresolved error')
|
|
275
|
+
.action(async (options) => {
|
|
276
|
+
const { paths } = await currentAthlete();
|
|
277
|
+
const runIds = Array.isArray(options.run) ? options.run : options.run ? [options.run] : undefined;
|
|
278
|
+
process.stdout.write(`${JSON.stringify(await resolveExtractionErrors(paths, { runIds, provider: options.provider, before: options.before, all: options.all }), null, 2)}\n`);
|
|
279
|
+
});
|
|
255
280
|
await program.parseAsync();
|
|
256
281
|
//# sourceMappingURL=commands.js.map
|
package/dist/runtime/index.d.ts
CHANGED
|
@@ -28,7 +28,7 @@ export { WELLNESS_METRICS, WellnessService } from '../core/query/wellness.js';
|
|
|
28
28
|
export { jsonSafe, ReadOnlyRepository } from '../core/query/repository.js';
|
|
29
29
|
export { queryReadOnlyData } from '../core/query/sql-guard.js';
|
|
30
30
|
export { createDemoStore, demoStoreMetadata } from '../elt/application/demo.js';
|
|
31
|
-
export { connectStrava, connectStravaWithCallback, dataStatus, disconnectStravaAccount, initializeDataStore, linkActivity, rebuildRetrievalIndex, retryDataSync, syncData, syncProgress, unlinkActivity, } from '../elt/application/management.js';
|
|
31
|
+
export { connectStrava, connectStravaWithCallback, dataStatus, disconnectStravaAccount, initializeDataStore, importStagedFile, linkActivity, rebuildRetrievalIndex, reimportNutrition, resolveExtractionErrors, retryDataSync, syncData, syncProgress, unlinkActivity, } from '../elt/application/management.js';
|
|
32
32
|
export type { ProviderChoice } from '../elt/application/management.js';
|
|
33
33
|
export type { InterruptedRuns, SyncProgressSnapshot, SyncProgressState } from '../contracts/progress.js';
|
|
34
34
|
export { openReadOnlyRepository, ReadOnlyDatabaseError } from '../elt/storage/database.js';
|
package/dist/runtime/index.js
CHANGED
|
@@ -15,7 +15,7 @@ export { WELLNESS_METRICS, WellnessService } from '../core/query/wellness.js';
|
|
|
15
15
|
export { jsonSafe, ReadOnlyRepository } from '../core/query/repository.js';
|
|
16
16
|
export { queryReadOnlyData } from '../core/query/sql-guard.js';
|
|
17
17
|
export { createDemoStore, demoStoreMetadata } from '../elt/application/demo.js';
|
|
18
|
-
export { connectStrava, connectStravaWithCallback, dataStatus, disconnectStravaAccount, initializeDataStore, linkActivity, rebuildRetrievalIndex, retryDataSync, syncData, syncProgress, unlinkActivity, } from '../elt/application/management.js';
|
|
18
|
+
export { connectStrava, connectStravaWithCallback, dataStatus, disconnectStravaAccount, initializeDataStore, importStagedFile, linkActivity, rebuildRetrievalIndex, reimportNutrition, resolveExtractionErrors, retryDataSync, syncData, syncProgress, unlinkActivity, } from '../elt/application/management.js';
|
|
19
19
|
export { openReadOnlyRepository, ReadOnlyDatabaseError } from '../elt/storage/database.js';
|
|
20
20
|
export { DataWriteBusyError } from '../elt/storage/write-lock.js';
|
|
21
21
|
export { hydrateStravaActivity, hydrateStravaSegmentHistory, StravaEnrichmentError, StravaRateLimitError, } from '../elt/ingestion/providers/strava/service.js';
|
package/package.json
CHANGED
|
@@ -183,6 +183,44 @@ def fit_archive_to_samples(activity_source_id: str, archive: Path) -> list[dict[
|
|
|
183
183
|
return []
|
|
184
184
|
|
|
185
185
|
|
|
186
|
+
def fit_archive_to_swim_lengths(activity_source_id: str, archive: Path) -> list[dict[str, Any]]:
|
|
187
|
+
"""Best-effort per-length parsing from the FIT 'length' messages; malformed/unavailable archives yield no rows."""
|
|
188
|
+
try:
|
|
189
|
+
import fitdecode
|
|
190
|
+
with zipfile.ZipFile(archive) as zip_file:
|
|
191
|
+
fit_name = next((name for name in zip_file.namelist() if name.lower().endswith(".fit")), None)
|
|
192
|
+
if not fit_name:
|
|
193
|
+
return []
|
|
194
|
+
with zip_file.open(fit_name) as stream, fitdecode.FitReader(stream) as reader:
|
|
195
|
+
rows: list[dict[str, Any]] = []
|
|
196
|
+
for frame in reader:
|
|
197
|
+
if not isinstance(frame, fitdecode.records.FitDataMessage) or frame.name != "length":
|
|
198
|
+
continue
|
|
199
|
+
fields = {field.name: field.value for field in frame.fields}
|
|
200
|
+
length_type = str(fields.get("length_type") or "").lower()
|
|
201
|
+
start_time = fields.get("start_time") or fields.get("timestamp")
|
|
202
|
+
row = {
|
|
203
|
+
"lengthIndex": _number(fields.get("message_index")),
|
|
204
|
+
"lapIndex": None,
|
|
205
|
+
"startTime": start_time.isoformat().replace("+00:00", "Z") if hasattr(start_time, "isoformat") else None,
|
|
206
|
+
"durationS": _number(fields.get("total_elapsed_time") or fields.get("total_timer_time")),
|
|
207
|
+
"activeDurationS": _number(fields.get("total_timer_time")),
|
|
208
|
+
"distanceM": _number(fields.get("distance")),
|
|
209
|
+
"strokeCount": _number(fields.get("total_strokes")),
|
|
210
|
+
"strokeRate": _number(fields.get("avg_swimming_cadence") or fields.get("avg_cadence")),
|
|
211
|
+
"swolf": _number(fields.get("avg_swolf")),
|
|
212
|
+
"avgHr": _number(fields.get("avg_heart_rate") or fields.get("average_heart_rate")),
|
|
213
|
+
"maxHr": _number(fields.get("max_heart_rate")),
|
|
214
|
+
"isRest": length_type in {"rest", "idle", "pause"},
|
|
215
|
+
"label": str(fields.get("swim_stroke") or "").lower() or None,
|
|
216
|
+
"raw": {name: (value.isoformat().replace("+00:00", "Z") if hasattr(value, "isoformat") else value) for name, value in fields.items()},
|
|
217
|
+
}
|
|
218
|
+
rows.append(row)
|
|
219
|
+
return rows
|
|
220
|
+
except (ImportError, OSError, ValueError, zipfile.BadZipFile):
|
|
221
|
+
return []
|
|
222
|
+
|
|
223
|
+
|
|
186
224
|
def write_parquet(data_dir: Path, activity_id: str, start_date: str, samples: Iterable[dict[str, Any]]) -> tuple[Path, str, int]:
|
|
187
225
|
records = list(samples)
|
|
188
226
|
year, month = (start_date[:4] or "unknown"), (start_date[5:7] or "unknown")
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from .staging import StagingWriter
|
|
7
|
+
from .streams import fit_archive_to_swim_lengths
|
|
8
|
+
|
|
9
|
+
SWIM_SPORTS = ("lap_swimming", "swimming", "open_water_swimming", "openwaterswimming")
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def parse_args() -> argparse.Namespace:
|
|
13
|
+
parser = argparse.ArgumentParser(description="Backfill swim lengths from already-archived Garmin activity FIT files.")
|
|
14
|
+
parser.add_argument("--data-dir", required=True, type=Path)
|
|
15
|
+
parser.add_argument("--output", required=True, type=Path)
|
|
16
|
+
return parser.parse_args()
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def main() -> None:
|
|
20
|
+
args = parse_args()
|
|
21
|
+
database_path = args.data_dir / "catence.duckdb"
|
|
22
|
+
if not database_path.exists():
|
|
23
|
+
raise SystemExit(f"Catalog database not found: {database_path}")
|
|
24
|
+
import duckdb
|
|
25
|
+
|
|
26
|
+
con = duckdb.connect(str(database_path), read_only=True)
|
|
27
|
+
rows = con.execute(
|
|
28
|
+
"""
|
|
29
|
+
SELECT source.remote_activity_id, cast(activity.started_at_utc AS VARCHAR) AS started_at_utc
|
|
30
|
+
FROM activity_sources AS source
|
|
31
|
+
JOIN activities AS activity USING (activity_id)
|
|
32
|
+
WHERE source.provider = 'garmin'
|
|
33
|
+
AND lower(coalesce(activity.sport, '')) IN ({placeholders})
|
|
34
|
+
ORDER BY activity.started_at_utc
|
|
35
|
+
""".format(placeholders=", ".join(f"'{sport}'" for sport in SWIM_SPORTS)),
|
|
36
|
+
).fetchall()
|
|
37
|
+
con.close()
|
|
38
|
+
|
|
39
|
+
writer = StagingWriter(args.data_dir, args.output)
|
|
40
|
+
staged = 0
|
|
41
|
+
for remote_id, started_at_utc in rows:
|
|
42
|
+
activity_id = str(remote_id)
|
|
43
|
+
zip_dir = args.data_dir / "raw" / "garmin" / "activity_original" / activity_id
|
|
44
|
+
if not zip_dir.is_dir():
|
|
45
|
+
continue
|
|
46
|
+
zips = sorted(zip_dir.glob("*.zip"))
|
|
47
|
+
if not zips:
|
|
48
|
+
continue
|
|
49
|
+
archive = zips[-1]
|
|
50
|
+
lengths = fit_archive_to_swim_lengths(f"garmin:{activity_id}", archive)
|
|
51
|
+
if not lengths:
|
|
52
|
+
continue
|
|
53
|
+
writer.source_entity(
|
|
54
|
+
"activity_swim_lengths",
|
|
55
|
+
f"{activity_id}:fit",
|
|
56
|
+
{"lengths": lengths},
|
|
57
|
+
archive.stem,
|
|
58
|
+
parent_remote_id=activity_id,
|
|
59
|
+
occurred_on=str(started_at_utc)[:10] if started_at_utc else None,
|
|
60
|
+
)
|
|
61
|
+
staged += 1
|
|
62
|
+
print(f"Staged swim lengths for {staged} activities; output: {args.output}")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
if __name__ == "__main__":
|
|
66
|
+
main()
|
|
@@ -9,7 +9,7 @@ from typing import Any, Callable, Mapping
|
|
|
9
9
|
from .progress import ProgressReporter
|
|
10
10
|
from .registry import GARMIN_READ_METHODS, assert_read_only_registry
|
|
11
11
|
from .staging import StagingWriter
|
|
12
|
-
from .streams import SAMPLE_COLUMNS, activity_details_to_samples, activity_power_bests, fit_archive_to_samples, write_parquet
|
|
12
|
+
from .streams import SAMPLE_COLUMNS, activity_details_to_samples, activity_power_bests, fit_archive_to_samples, fit_archive_to_swim_lengths, write_parquet
|
|
13
13
|
|
|
14
14
|
|
|
15
15
|
class WorkerInterrupted(Exception):
|
|
@@ -556,6 +556,20 @@ class GarminStagingWorker:
|
|
|
556
556
|
parent_remote_id=activity_id,
|
|
557
557
|
occurred_on=start_date[:10],
|
|
558
558
|
)
|
|
559
|
+
if original_path and original_path.exists():
|
|
560
|
+
sport = str(summary.get("sportType") or summary.get("sport") or "").lower()
|
|
561
|
+
if sport == "lap_swimming" or "swim" in sport:
|
|
562
|
+
lengths = fit_archive_to_swim_lengths(f"garmin:{activity_id}", original_path)
|
|
563
|
+
if lengths:
|
|
564
|
+
start_date = str(summary.get("startTimeLocal") or summary.get("startTimeGMT") or date.today().isoformat())
|
|
565
|
+
self.writer.source_entity(
|
|
566
|
+
"activity_swim_lengths",
|
|
567
|
+
f"{activity_id}:fit",
|
|
568
|
+
{"lengths": lengths},
|
|
569
|
+
original_hash,
|
|
570
|
+
parent_remote_id=activity_id,
|
|
571
|
+
occurred_on=start_date[:10],
|
|
572
|
+
)
|
|
559
573
|
|
|
560
574
|
def _collections(self) -> None:
|
|
561
575
|
today = date.today()
|