catence 0.2.0-beta.11 → 0.2.0-beta.13

package/dist/contracts/progress.d.ts

@@ -1,5 +1,5 @@
 export declare const STALE_RUN_TIMEOUT_MS: number;
-export declare const SYNC_STAGES: readonly ["starting", "login", "singletons", "daily", "range", "ftp_history", "max_metrics", "hrv_history", "scores", "activities", "collections", "importing", "completed", "interrupted", "failed", "timed_out"];
+export declare const SYNC_STAGES: readonly ["starting", "login", "singletons", "daily", "range", "ftp_history", "max_metrics", "hrv_history", "scores", "activities", "collections", "intervals", "importing", "completed", "interrupted", "failed", "timed_out"];
 export type SyncStage = (typeof SYNC_STAGES)[number];
 export interface SyncProgressState {
     runId: string;

package/dist/contracts/progress.js

@@ -11,6 +11,7 @@ export const SYNC_STAGES = [
     'scores',
     'activities',
     'collections',
+    'intervals',
     'importing',
     'completed',
     'interrupted',

package/dist/contracts/release.d.ts

@@ -3,7 +3,7 @@
  * Keep these values synchronized with release/manifest.json; release checks
  * deliberately reject drift before a package can be published.
  */
-export declare const CATENCE_RUNTIME_VERSION = "0.2.0-beta.11";
+export declare const CATENCE_RUNTIME_VERSION = "0.2.0-beta.13";
 export declare const CATENCE_PROTOCOL_VERSION = 1;
 export type CatenceRuntimeCapabilities = {
     mcp: true;

package/dist/contracts/release.js

@@ -3,7 +3,7 @@
  * Keep these values synchronized with release/manifest.json; release checks
  * deliberately reject drift before a package can be published.
  */
-export const CATENCE_RUNTIME_VERSION = '0.2.0-beta.11';
+export const CATENCE_RUNTIME_VERSION = '0.2.0-beta.13';
 export const CATENCE_PROTOCOL_VERSION = 1;
 export function catenceRuntimeHealth() {
     return {

package/dist/elt/application/management.d.ts

@@ -24,4 +24,17 @@ export declare function unlinkActivity(paths: CatencePaths, source: string): Pro
 export declare function dataStatus(paths: CatencePaths): Promise<Record<string, unknown>>;
 export declare function syncProgress(paths: CatencePaths): Promise<SyncProgressSnapshot>;
 export declare function rebuildRetrievalIndex(paths: CatencePaths): Promise<Record<string, unknown>>;
+export declare function importStagedFile(paths: CatencePaths, provider: ProviderChoice, filePath: string): Promise<Record<string, unknown>>;
+export declare function resolveExtractionErrors(paths: CatencePaths, opts?: {
+    runIds?: string[];
+    provider?: string;
+    before?: string;
+    all?: boolean;
+}): Promise<Record<string, unknown>>;
+/**
+ * Re-run normalization over already-captured nutrition entities without
+ * contacting any provider. Idempotent upserts let a fixed normalizer backfill
+ * previously mis-parsed nutrition_days/nutrition_items from stored payloads.
+ */
+export declare function reimportNutrition(paths: CatencePaths): Promise<Record<string, unknown>>;
 export {};

package/dist/elt/application/management.js

@@ -14,8 +14,9 @@ import { importJsonl } from '../ingestion/importer.js';
 import { IntervalsExtractor } from '../ingestion/providers/intervals.js';
 import { completeStravaAuthorization, disconnectStrava, getStravaAuthorizationUrl, syncStravaGear } from '../ingestion/providers/strava/service.js';
 import { setManualActivityLink, unlinkActivitySource } from '../normalization/activities/linking.js';
+import { importSourceEntity } from '../normalization/normalizers.js';
 import { CatenceDatabase, openReadOnlyRepository, ReadOnlyDatabaseError } from '../storage/database.js';
-import { mergeProgress, readRunningProgress, removeProgressSidecar, writeProgressSidecar } from '../storage/progress-sidecar.js';
+import { mergeProgress, ProgressPump, readRunningProgress, removeProgressSidecar } from '../storage/progress-sidecar.js';
 import { withDataWriteLock } from '../storage/write-lock.js';
 function packageRoot() {
     return path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../..');
@@ -33,7 +34,7 @@ function baseProgress(runId, provider, stage) {
  * Progress NDJSON lines (kind 'progress') are normalized and persisted as
  * throttled heartbeats; other output is forwarded to the sync logger.
  */
-function runGarminWorker(runId, arguments_, environment, database, paths, log, onInterruptReady) {
+function runGarminWorker(runId, arguments_, environment, pump, log, onInterruptReady) {
     return new Promise((resolve) => {
         const child = spawn('uv', arguments_, { ...environment, stdio: ['ignore', 'pipe', 'pipe'] });
         let killTimer;
@@ -48,24 +49,6 @@ function runGarminWorker(runId, arguments_, environment, database, paths, log, o
             }, 10_000);
             killTimer.unref();
         });
-        let lastProgress = null;
-        let lastHeartbeatAt = 0;
-        const heartbeat = (state) => {
-            const now = Date.now();
-            if (now - lastHeartbeatAt < 2_000)
-                return;
-            lastHeartbeatAt = now;
-            void database.heartbeatRun(runId, state).catch((error) => {
-                log.debug('Progress heartbeat failed', { runId, error: error instanceof Error ? error.message : String(error) });
-            });
-            void writeProgressSidecar(paths, runId, state).catch((error) => {
-                log.debug('Progress sidecar write failed', { runId, error: error instanceof Error ? error.message : String(error) });
-            });
-        };
-        const fallback = setInterval(() => {
-            heartbeat(lastProgress ?? baseProgress(runId, 'garmin', 'starting'));
-        }, 30_000);
-        fallback.unref();
         const buffer = (stream, sink) => {
             let pending = '';
             stream.on('data', (chunk) => {
@@ -95,29 +78,24 @@ function runGarminWorker(runId, arguments_, environment, database, paths, log, o
             }
             if (typeof parsed === 'object' && parsed !== null && parsed.kind === 'progress') {
                 const state = normalizeProgress(parsed);
-                if (state) {
-                    lastProgress = state;
-                    heartbeat(state);
-                }
+                if (state)
+                    pump.publish(state);
                 return;
             }
             log.info(`[garmin] ${line}`);
         });
         buffer(child.stderr, (line) => log.error(`[garmin] ${line}`));
         child.on('error', (error) => {
-            clearInterval(fallback);
             if (killTimer)
                 clearTimeout(killTimer);
             resolve({ code: 1, signal: null });
             log.error('Garmin staging worker could not be started', { runId, error: error.message });
         });
         child.on('close', (code, signal) => {
-            clearInterval(fallback);
             if (killTimer)
                 clearTimeout(killTimer);
-            if (code === 0 && lastProgress) {
-                void writeProgressSidecar(paths, runId, { ...lastProgress, stage: 'importing' }).catch(() => undefined);
-            }
+            if (code === 0)
+                pump.setStageImporting();
             resolve({ code, signal });
         });
     });
@@ -188,11 +166,14 @@ async function syncProvider(database, paths, provider, options, log) {
     });
     let extractionStarted = false;
     const guard = { interrupted: false };
+    const pump = new ProgressPump(runId, provider, { database, paths, log });
+    pump.start();
     try {
         if (provider === 'intervals') {
             const environment = await athleteProviderEnvironment(paths);
             const config = requireIntervalsConfig(environment);
             extractionStarted = true;
+            pump.setStage('intervals');
             await runWithInterruptGuard(runId, database, () => undefined, guard, async () => {
                 await new IntervalsExtractor(database, paths, config.apiKey, config.athleteId, log).sync(runId, activityWindow, !options.backfill || options.refreshActivities);
             });
@@ -222,7 +203,7 @@ async function syncProvider(database, paths, provider, options, log) {
                 const outcome = await runGarminWorker(runId, arguments_, {
                     cwd: packageRoot(),
                     env: { ...(await athleteProviderEnvironment(paths)), UV_PROJECT_ENVIRONMENT: process.env.UV_PROJECT_ENVIRONMENT ?? path.join(paths.root, 'python-venv') },
-                }, database, paths, log, (kill) => { interruptChild = kill; });
+                }, pump, log, (kill) => { interruptChild = kill; });
                 if (!guard.interrupted && outcome.code !== 0) {
                     throw new Error(`Garmin staging worker exited with code ${outcome.code}${outcome.signal ? ` (${outcome.signal})` : ''}.`);
                 }
@@ -243,12 +224,14 @@ async function syncProvider(database, paths, provider, options, log) {
         }
     }
     finally {
+        pump.stop();
         if (guard.interrupted) {
             await database.markRunInterrupted(runId);
             log.warn('Sync run interrupted', { runId, provider });
         }
         else {
             await database.finishRun(runId);
+            pump.publishFinal(baseProgress(runId, provider, 'completed'));
         }
         await removeProgressSidecar(paths, runId).catch(() => undefined);
         const [run] = await database.rows('SELECT status, error_count FROM sync_runs WHERE run_id = $runId', { runId });
@@ -291,6 +274,11 @@ export async function syncData(paths, provider, explicitFrom, advanceCursor = tr
         const runs = [];
         for (const source of providers)
             runs.push(await syncProvider(database, paths, source, { explicitFrom, toDate, advanceCursor, refreshActivities, backfill }, log));
+        const executedRuns = runs.filter((run) => !run.skipped);
+        if (executedRuns.length > 0) {
+            const index = await buildRetrievalIndex(database);
+            log.info('Rebuilt retrieval index after sync', { documents: index.documents, mode: index.mode, watermark: index.watermark });
+        }
         return { runIds: runs.flatMap((run) => run.runId ? [run.runId] : []), runs, timedOutRuns: timedOutRuns.runIds };
     }
     finally {
@@ -484,4 +472,76 @@ export async function rebuildRetrievalIndex(paths) {
         await database.close();
     }
 }
+export async function importStagedFile(paths, provider, filePath) {
+    if (!existsSync(paths.database))
+        throw new Error('No Catence database exists yet. Complete a sync before importing staged records.');
+    if (!existsSync(filePath))
+        throw new Error(`Staging file not found: ${filePath}`);
+    const database = await CatenceDatabase.open(paths);
+    try {
+        const runId = randomUUID();
+        const importedRecords = await importJsonl(database, runId, filePath);
+        return { runId, provider, importedRecords };
+    }
+    finally {
+        await database.close();
+    }
+}
+export async function resolveExtractionErrors(paths, opts = {}) {
+    if (!existsSync(paths.database))
+        throw new Error('No Catence database exists yet.');
+    const database = await CatenceDatabase.open(paths);
+    try {
+        const resolved = await database.resolveErrors({
+            runIds: opts.runIds,
+            provider: opts.provider,
+            before: opts.before,
+            all: opts.all,
+        });
+        return { resolvedErrors: resolved };
+    }
+    finally {
+        await database.close();
+    }
+}
+/**
+ * Re-run normalization over already-captured nutrition entities without
+ * contacting any provider. Idempotent upserts let a fixed normalizer backfill
+ * previously mis-parsed nutrition_days/nutrition_items from stored payloads.
+ */
+export async function reimportNutrition(paths) {
+    if (!existsSync(paths.database))
+        throw new Error('No Catence database exists yet. Complete a sync before re-importing nutrition.');
+    const database = await CatenceDatabase.open(paths);
+    try {
+        const rows = await database.rows(`SELECT provider, entity_type, remote_id, parent_remote_id, occurred_on, source_updated_at, raw_object_hash,
+              cast(payload_json AS VARCHAR) AS payload_json, cast(extension_json AS VARCHAR) AS extension_json
+       FROM source_entities
+       WHERE entity_type IN ('nutrition_log', 'nutrition_day')
+       ORDER BY provider, occurred_on, remote_id`);
+        for (const row of rows) {
+            await importSourceEntity(database, {
+                kind: 'source_entity', schemaVersion: 1, provider: row.provider, entityType: row.entity_type,
+                remoteId: row.remote_id, parentRemoteId: row.parent_remote_id, occurredOn: row.occurred_on, sourceUpdatedAt: row.source_updated_at,
+                rawObjectHash: row.raw_object_hash, payload: parseJsonObject(row.payload_json), extension: parseJsonObject(row.extension_json),
+            });
+        }
+        const days = (await database.rows('SELECT count(*) AS count FROM nutrition_days'))[0]?.count ?? 0;
+        const items = (await database.rows('SELECT count(*) AS count FROM nutrition_items'))[0]?.count ?? 0;
+        return { reimportedEntities: rows.length, nutritionDays: days, nutritionItems: items };
+    }
+    finally {
+        await database.close();
+    }
+}
+function parseJsonObject(value) {
+    if (!value)
+        return {};
+    try {
+        return JSON.parse(value);
+    }
+    catch {
+        return {};
+    }
+}
 //# sourceMappingURL=management.js.map

package/dist/elt/ingestion/importer.js

@@ -33,7 +33,7 @@ export async function importRecord(database, runId, record, log) {
             if (activity[0]?.activity_id)
                 await refreshActivityQuality(database, activity[0].activity_id);
         }
-        if (record.provider === 'garmin' && ['activity_detail', 'activity_interval', 'activity_exercise_set'].includes(record.entityType) && record.parentRemoteId) {
+        if (record.provider === 'garmin' && ['activity_detail', 'activity_interval', 'activity_exercise_set', 'activity_swim_lengths'].includes(record.entityType) && record.parentRemoteId) {
             const activity = await database.rows('SELECT activity_id FROM activity_sources WHERE activity_source_id = $activitySourceId', { activitySourceId: `garmin:${record.parentRemoteId}` });
             if (activity[0]?.activity_id)
                 await refreshActivityQuality(database, activity[0].activity_id);

package/dist/elt/normalization/normalizers.js

@@ -163,24 +163,63 @@ async function upsertDailyMetric(database, provider, metricDate, metricName, val
      ON CONFLICT (provider, metric_date, metric_name) DO UPDATE SET value_number = excluded.value_number, value_text = excluded.value_text,
        unit = excluded.unit, raw_object_hash = excluded.raw_object_hash`, { provider, metricDate, metricName, valueNumber, valueText, unit, rawHash });
 }
+/**
+ * Garmin's nutrition food log keeps its actuals under dailyNutritionContent
+ * (a per-day rolled-up object), while meal definitions and hydration endpoints
+ * share the same source entity type. Flat keys are kept for providers that
+ * return them directly.
+ */
 function nutritionFields(payload) {
+    const content = object(payload.dailyNutritionContent);
     return {
-        energyKcal: firstNumber(payload, ['kcalConsumed', 'calories', 'totalCalories', 'energyKcal']),
-        carbohydratesG: firstNumber(payload, ['carbohydrates', 'carbs', 'totalCarbs', 'carbohydrateGrams']),
-        proteinG: firstNumber(payload, ['protein', 'totalProtein', 'proteinGrams']),
-        fatG: firstNumber(payload, ['fatTotal', 'fat', 'totalFat', 'fatGrams']),
-        hydrationMl: firstNumber(payload, ['hydrationVolume', 'hydration', 'waterMl', 'totalWater']),
+        energyKcal: firstNumber(payload, ['kcalConsumed', 'calories', 'totalCalories', 'energyKcal']) ?? firstNumber(content, ['calories', 'energyKcal', 'totalCalories']),
+        carbohydratesG: firstNumber(payload, ['carbohydrates', 'carbs', 'totalCarbs', 'carbohydrateGrams']) ?? firstNumber(content, ['carbohydrates', 'carbs']),
+        proteinG: firstNumber(payload, ['protein', 'totalProtein', 'proteinGrams']) ?? firstNumber(content, ['protein']),
+        fatG: firstNumber(payload, ['fatTotal', 'fat', 'totalFat', 'fatGrams']) ?? firstNumber(content, ['fat', 'fatTotal']),
+        hydrationMl: firstNumber(payload, ['hydrationVolume', 'hydration', 'waterMl', 'totalWater', 'valueInML', 'dailyAverageinML']),
     };
 }
+/**
+ * Garmin food-log items live under mealDetails[].foods[]. Each food carries the
+ * day's nutrition_date, so a meal label is folded onto the item to avoid losing
+ * it. Flat arrays are kept for providers that return items directly.
+ */
 function nutritionItems(payload) {
-    const candidates = ['foodItems', 'items', 'entries', 'foodLogEntries', 'meals'];
-    for (const key of candidates) {
+    const items = [];
+    const mealDetails = payload.mealDetails;
+    if (Array.isArray(mealDetails)) {
+        for (const detail of mealDetails) {
+            const mealDetail = object(detail);
+            const mealName = firstString(object(mealDetail.meal), ['mealName']) ?? firstString(mealDetail, ['mealName']);
+            const foods = mealDetail.foods;
+            if (Array.isArray(foods)) {
+                for (const food of foods) {
+                    const item = object(food);
+                    if (Object.keys(item).length === 0)
+                        continue;
+                    items.push(mealName ? { ...item, mealName } : item);
+                }
+            }
+            else if (Object.keys(mealDetail).length > 0 && firstString(mealDetail, ['foodName', 'name', 'description']) !== null) {
+                items.push(mealName ? { ...mealDetail, mealName } : mealDetail);
+            }
+        }
+        if (items.length > 0)
+            return items;
+    }
+    for (const key of ['foodItems', 'items', 'entries', 'foodLogEntries', 'meals']) {
         const value = payload[key];
         if (Array.isArray(value))
             return value.map(object).filter((item) => Object.keys(item).length > 0);
     }
     return [];
 }
+/** True when a payload contributes real nutrition facts rather than meal definitions or hydration-only bookkeeping. */
+function hasNutritionContent(payload) {
+    const fields = nutritionFields(payload);
+    return fields.energyKcal !== null || fields.carbohydratesG !== null || fields.proteinG !== null || fields.fatG !== null
+        || fields.hydrationMl !== null || nutritionItems(payload).length > 0;
+}
 export async function importSourceEntity(database, entity) {
     const payload = object(entity.payload);
     const rawHash = entity.rawObjectHash;
@@ -223,6 +262,9 @@ export async function importSourceEntity(database, entity) {
     if (entity.provider === 'garmin' && ['activity_detail', 'activity_interval', 'activity_exercise_set'].includes(entity.entityType)) {
         await importGarminExplicitSwimLengths(database, entity.parentRemoteId, entity.entityType, payload, rawHash);
     }
+    if (entity.provider === 'garmin' && entity.entityType === 'activity_swim_lengths') {
+        await importGarminFitSwimLengths(database, entity.parentRemoteId, payload, rawHash);
+    }
     if (entity.entityType === 'wellness' || entity.entityType === 'daily_health')
         await importDailyMetrics(database, entity.provider, entity.remoteId, entity.occurredOn, payload, rawHash);
     if (entity.entityType === 'nutrition_day' || entity.entityType === 'nutrition_log')
@@ -668,6 +710,46 @@ async function importGarminExplicitSwimLengths(database, parentRemoteId, entityT
         }
     }
 }
+async function importGarminFitSwimLengths(database, parentRemoteId, payload, rawHash) {
+    if (!parentRemoteId)
+        return;
+    const activitySourceId = providerActivityId('garmin', parentRemoteId);
+    const lengths = objectArray(payload.lengths);
+    if (lengths.length === 0)
+        return;
+    const source = 'garmin_fit_length';
+    await database.run('DELETE FROM swim_lengths WHERE activity_source_id = $activitySourceId AND source = $source', { activitySourceId, source });
+    const summaryPoolLengthM = await activityPoolLengthM(database, activitySourceId);
+    for (const [offset, length] of lengths.entries()) {
+        const durationS = firstNumber(length, ['durationS', 'duration', 'durationSeconds', 'elapsedDuration']);
+        if (durationS === null)
+            continue;
+        const sourceLabel = swimSourceLabel(length);
+        const isRest = firstBoolean(length, ['isRest', 'rest']) ?? Boolean(sourceLabel && /rest|idle|pause/.test(sourceLabel));
+        const lengthIndex = Math.round(firstNumber(length, ['lengthIndex', 'messageIndex', 'lengthNumber', 'index']) ?? offset);
+        await database.run(`INSERT INTO swim_lengths
+        (activity_source_id, source, length_index, lap_index, pool_length_m, start_time, duration_s, active_duration_s, distance_m, stroke_count, stroke_rate, swolf, avg_hr, max_hr, is_rest, confidence, metrics_json, raw_object_hash)
+       VALUES ($activitySourceId, $source, $lengthIndex, $lapIndex, $poolLengthM, $startTime, $durationS, $activeDurationS, $distanceM, $strokeCount, $strokeRate, $swolf, $avgHr, $maxHr, $isRest, 'provider_supplied', $metrics, $rawHash)
+       ON CONFLICT (activity_source_id, source, length_index) DO UPDATE SET
+         lap_index = excluded.lap_index, pool_length_m = excluded.pool_length_m, start_time = excluded.start_time,
+         duration_s = excluded.duration_s, active_duration_s = excluded.active_duration_s, distance_m = excluded.distance_m,
+         stroke_count = excluded.stroke_count, stroke_rate = excluded.stroke_rate, swolf = excluded.swolf,
+         avg_hr = excluded.avg_hr, max_hr = excluded.max_hr, is_rest = excluded.is_rest,
+         confidence = excluded.confidence, metrics_json = excluded.metrics_json, raw_object_hash = excluded.raw_object_hash`, {
+            activitySourceId, source, lengthIndex,
+            lapIndex: firstNumber(length, ['lapIndex', 'lapNumber']), poolLengthM: garminPoolLengthM(length) ?? summaryPoolLengthM,
+            startTime: timestamp(length.startTime ?? length.startTimestamp ?? length.timestamp), durationS,
+            activeDurationS: firstNumber(length, ['activeDurationS', 'activeDuration', 'movingDuration']),
+            distanceM: firstNumber(length, ['distanceM', 'distance']),
+            strokeCount: firstNumber(length, ['strokeCount', 'strokes', 'total_strokes']),
+            strokeRate: firstNumber(length, ['strokeRate', 'averageStrokeRate', 'avg_swimming_cadence', 'avg_cadence', 'cadence']),
+            swolf: firstNumber(length, ['swolf', 'avg_swolf']),
+            avgHr: firstNumber(length, ['avgHr', 'averageHR', 'avg_heart_rate']),
+            maxHr: firstNumber(length, ['maxHr', 'maxHR', 'max_heart_rate']),
+            isRest, metrics: json(length), rawHash,
+        });
+    }
+}
 async function addActivityQualityFlag(database, activitySourceId, code, severity, details, rawHash) {
     await database.run(`INSERT INTO activity_quality_flags (activity_source_id, flag_code, severity, details_json, raw_object_hash)
      VALUES ($activitySourceId, $code, $severity, $details, $rawHash)
@@ -791,12 +873,19 @@ async function importNutrition(database, provider, date, remoteId, payload, rawH
     const nutritionDate = datePart(date) ?? datePart(firstString(payload, ['id', 'date', 'calendarDate'])) ?? null;
     if (!nutritionDate)
         return;
+    if (!hasNutritionContent(payload))
+        return;
     const fields = nutritionFields(payload);
     await database.run(`INSERT INTO nutrition_days
       (provider, nutrition_date, energy_kcal, carbohydrates_g, protein_g, fat_g, hydration_ml, metrics_json, raw_object_hash)
       VALUES ($provider, $nutritionDate, $energyKcal, $carbohydratesG, $proteinG, $fatG, $hydrationMl, $metrics, $rawHash)
-      ON CONFLICT (provider, nutrition_date) DO UPDATE SET energy_kcal = excluded.energy_kcal, carbohydrates_g = excluded.carbohydrates_g,
-       protein_g = excluded.protein_g, fat_g = excluded.fat_g, hydration_ml = excluded.hydration_ml, metrics_json = excluded.metrics_json, raw_object_hash = excluded.raw_object_hash`, { provider, nutritionDate, ...fields, metrics: json(payload), rawHash });
+      ON CONFLICT (provider, nutrition_date) DO UPDATE SET
+        energy_kcal = COALESCE(excluded.energy_kcal, nutrition_days.energy_kcal),
+        carbohydrates_g = COALESCE(excluded.carbohydrates_g, nutrition_days.carbohydrates_g),
+        protein_g = COALESCE(excluded.protein_g, nutrition_days.protein_g),
+        fat_g = COALESCE(excluded.fat_g, nutrition_days.fat_g),
+        hydration_ml = COALESCE(excluded.hydration_ml, nutrition_days.hydration_ml),
+        metrics_json = excluded.metrics_json, raw_object_hash = excluded.raw_object_hash`, { provider, nutritionDate, ...fields, metrics: json(payload), rawHash });
     for (const [index, item] of nutritionItems(payload).entries()) {
         const itemFields = nutritionFields(item);
         const itemId = firstString(item, ['id', 'foodId', 'entryId']) ?? `${remoteId}:${index}`;

package/dist/elt/storage/database.d.ts

@@ -22,6 +22,12 @@ export declare class CatenceDatabase implements WriteDataStore {
     interruptStaleSyncRuns(timeoutMs?: number): Promise<InterruptedRuns>;
     syncProgress(): Promise<SyncProgressSnapshot>;
     addError(runId: string, provider: Provider, endpoint: string, remoteId: string | null, message: string, retryable: boolean): Promise<void>;
+    resolveErrors(opts?: {
+        runIds?: string[];
+        provider?: Provider;
+        before?: string;
+        all?: boolean;
+    }): Promise<number>;
     status(): Promise<Record<string, unknown>>;
     getRun(runId: string): Promise<{
         provider: Provider;

package/dist/elt/storage/database.js

@@ -147,6 +147,28 @@ export class CatenceDatabase {
        VALUES ($errorId, $runId, $provider, $endpoint, $remoteId, $message, $retryable)`, { errorId: randomUUID(), runId, provider, endpoint, remoteId, message, retryable });
         await this.run('UPDATE sync_runs SET error_count = error_count + 1 WHERE run_id = $runId', { runId });
     }
+    async resolveErrors(opts = {}) {
+        const clauses = ['resolved_at IS NULL'];
+        const values = {};
+        if (opts.runIds && opts.runIds.length > 0) {
+            const placeholders = opts.runIds.map((_, index) => `$runId${index}`).join(', ');
+            clauses.push(`run_id IN (${placeholders})`);
+            opts.runIds.forEach((runId, index) => { values[`runId${index}`] = runId; });
+        }
+        if (opts.provider) {
+            clauses.push('provider = $provider');
+            values.provider = opts.provider;
+        }
+        if (opts.before) {
+            clauses.push('created_at < $before');
+            values.before = opts.before;
+        }
+        if (!opts.all && clauses.length === 1) {
+            throw new Error('Refusing to resolve all unresolved errors without an explicit --all flag.');
+        }
+        const resolved = await this.rows(`UPDATE normalization_errors SET resolved_at = now() WHERE ${clauses.join(' AND ')} RETURNING error_id`, values);
+        return resolved.length;
+    }
     async status() {
         // @duckdb/node-api owns native statement state on the connection. Keep
         // status reads sequential instead of issuing several native calls at once.

package/dist/elt/storage/progress-sidecar.d.ts

@@ -1,7 +1,11 @@
-import { type SyncProgressSnapshot, type SyncProgressState } from '../../contracts/progress.js';
+import { type SyncProgressSnapshot, type SyncProgressState, type SyncStage } from '../../contracts/progress.js';
 import type { CatencePaths } from '../../contracts/runtime.js';
+import type { SyncLogger } from '../../core/logging.js';
+import type { CatenceDatabase } from './database.js';
 export declare const PROGRESS_SIDECAR_SUFFIX = ".progress.json";
 export declare const PROGRESS_SIDECAR_STALE_MS: number;
+export declare const PROGRESS_KEEPALIVE_MS: number;
+export declare const PROGRESS_PUBLISH_THROTTLE_MS: number;
 export declare function progressSidecarPath(paths: CatencePaths, runId: string): string;
 /**
  * Persist one progress heartbeat as a sidecar file next to the Garmin staging
@@ -24,3 +28,34 @@ export declare function readRunningProgress(paths: CatencePaths, staleAfterMs?:
  * both sources and sorted by heartbeat recency.
  */
 export declare function mergeProgress(dbRunning: SyncProgressState[], dbRecent: SyncProgressState[], sidecars: SyncProgressState[]): SyncProgressSnapshot;
+export interface ProgressPumpOptions {
+    database: CatenceDatabase;
+    paths: CatencePaths;
+    log: SyncLogger;
+}
+/**
+ * Drives progress heartbeats for the whole lifetime of a sync run - including
+ * phases that emit no worker output (the Intervals extractor and the Garmin
+ * JSONL import). `publish` throttles writes to the database and sidecar; a
+ * 30-second keep-alive interval refreshes the last known state so a run never
+ * looks stale while it is still active, even during the lock-holding import.
+ */
+export declare class ProgressPump {
+    private readonly runId;
+    private readonly provider;
+    private readonly options;
+    private lastProgress;
+    private lastHeartbeatAt;
+    private interval;
+    private stageOverride;
+    constructor(runId: string, provider: string, options: ProgressPumpOptions);
+    start(): void;
+    publish(state: SyncProgressState): void;
+    /** Force subsequent keep-alive heartbeats to report the given stage. */
+    setStage(stage: SyncStage): void;
+    /** Transition the run to the JSONL import stage; keep-alives persist it. */
+    setStageImporting(): void;
+    /** Write a terminal heartbeat immediately, bypassing the publish throttle. */
+    publishFinal(state: SyncProgressState): void;
+    stop(): void;
+}

package/dist/elt/storage/progress-sidecar.js

@@ -1,9 +1,11 @@
 import { existsSync } from 'node:fs';
 import { mkdir, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises';
 import path from 'node:path';
-import { normalizeProgress } from '../../contracts/progress.js';
+import { normalizeProgress, } from '../../contracts/progress.js';
 export const PROGRESS_SIDECAR_SUFFIX = '.progress.json';
 export const PROGRESS_SIDECAR_STALE_MS = 5 * 60 * 1000;
+export const PROGRESS_KEEPALIVE_MS = 30 * 1000;
+export const PROGRESS_PUBLISH_THROTTLE_MS = 2 * 1000;
 const TERMINAL_STAGES = new Set(['completed', 'failed', 'interrupted', 'timed_out']);
 export function progressSidecarPath(paths, runId) {
     return path.join(paths.staging, 'garmin', `${runId}${PROGRESS_SIDECAR_SUFFIX}`);
@@ -16,7 +18,7 @@ export function progressSidecarPath(paths, runId) {
 export async function writeProgressSidecar(paths, runId, state) {
     const target = progressSidecarPath(paths, runId);
     await mkdir(path.dirname(target), { recursive: true });
-    const temporary = `${target}.tmp`;
+    const temporary = `${target}.${Date.now()}.${Math.random().toString(36).slice(2, 10)}.tmp`;
     await writeFile(temporary, JSON.stringify(state));
     await rename(temporary, target);
 }
@@ -84,4 +86,95 @@ export function mergeProgress(dbRunning, dbRecent, sidecars) {
             .slice(0, 10),
     };
 }
+function baselineState(runId, provider, stage) {
+    return {
+        runId,
+        provider,
+        stage,
+        currentStep: null,
+        completedUnits: 0,
+        totalUnits: null,
+        percentComplete: 0,
+        elapsedSeconds: 0,
+        estimatedRemainingSeconds: null,
+        heartbeatAt: new Date().toISOString(),
+    };
+}
+/**
+ * Drives progress heartbeats for the whole lifetime of a sync run - including
+ * phases that emit no worker output (the Intervals extractor and the Garmin
+ * JSONL import). `publish` throttles writes to the database and sidecar; a
+ * 30-second keep-alive interval refreshes the last known state so a run never
+ * looks stale while it is still active, even during the lock-holding import.
+ */
+export class ProgressPump {
+    runId;
+    provider;
+    options;
+    lastProgress = null;
+    lastHeartbeatAt = 0;
+    interval = null;
+    stageOverride = null;
+    constructor(runId, provider, options) {
+        this.runId = runId;
+        this.provider = provider;
+        this.options = options;
+    }
+    start() {
+        if (this.interval)
+            return;
+        this.interval = setInterval(() => {
+            this.publish(this.lastProgress
+                ? { ...this.lastProgress, heartbeatAt: new Date().toISOString() }
+                : baselineState(this.runId, this.provider, this.stageOverride ?? 'starting'));
+        }, PROGRESS_KEEPALIVE_MS);
+        this.interval.unref();
+    }
+    publish(state) {
+        const now = Date.now();
+        if (this.lastHeartbeatAt !== 0 && now - this.lastHeartbeatAt < PROGRESS_PUBLISH_THROTTLE_MS) {
+            this.lastProgress = state;
+            return;
+        }
+        this.lastHeartbeatAt = now;
+        this.lastProgress = state;
+        const { database, paths, log } = this.options;
+        void database.heartbeatRun(this.runId, state).catch((error) => {
+            log.debug('Progress heartbeat failed', { runId: this.runId, error: error instanceof Error ? error.message : String(error) });
+        });
+        void writeProgressSidecar(paths, this.runId, state).catch((error) => {
+            log.debug('Progress sidecar write failed', { runId: this.runId, error: error instanceof Error ? error.message : String(error) });
+        });
+    }
+    /** Force subsequent keep-alive heartbeats to report the given stage. */
+    setStage(stage) {
+        this.stageOverride = stage;
+        if (this.lastProgress) {
+            this.lastProgress = { ...this.lastProgress, stage };
+            this.publish(this.lastProgress);
+        }
+    }
+    /** Transition the run to the JSONL import stage; keep-alives persist it. */
+    setStageImporting() {
+        this.setStage('importing');
+    }
+    /** Write a terminal heartbeat immediately, bypassing the publish throttle. */
+    publishFinal(state) {
+        this.lastHeartbeatAt = Date.now();
+        this.lastProgress = state;
+        const { database, paths, log } = this.options;
+        void database.heartbeatRun(this.runId, state).catch((error) => {
+            log.debug('Progress heartbeat failed', { runId: this.runId, error: error instanceof Error ? error.message : String(error) });
+        });
+        void writeProgressSidecar(paths, this.runId, state).catch((error) => {
+            log.debug('Progress sidecar write failed', { runId: this.runId, error: error instanceof Error ? error.message : String(error) });
+        });
+    }
+    stop() {
+        if (this.interval) {
+            clearInterval(this.interval);
+            this.interval = null;
+        }
+    }
+}
 //# sourceMappingURL=progress-sidecar.js.map

package/dist/interfaces/cli/commands.js

@@ -2,7 +2,7 @@
 import { createInterface } from 'node:readline/promises';
 import { stdin as input, stdout as output } from 'node:process';
 import { Command } from 'commander';
-import { addAthlete, athleteStorePaths, CATENCE_RUNTIME_VERSION, channelForVersion, createDemoStore, dataStatus, defaultCatalogHome, disconnectStravaAccount, fetchNpmDistTags, fetchPypiReleases, hasCommand, initializeCatalog, isGlobalNpmInstall, linkActivity, loadCatalog, npmGlobalPrefix, planSelfUpdate, readInstalledConsoleVersion, rebuildRetrievalIndex, resolveAthlete, resolveCatalogPaths, retryDataSync, runSelfUpdateCommand, setAthleteSecret, syncData, syncProgress, connectStrava, connectStravaWithCallback, unlinkActivity, } from '../../runtime/index.js';
+import { addAthlete, athleteStorePaths, CATENCE_RUNTIME_VERSION, channelForVersion, createDemoStore, dataStatus, defaultCatalogHome, disconnectStravaAccount, fetchNpmDistTags, fetchPypiReleases, hasCommand, importStagedFile, initializeCatalog, isGlobalNpmInstall, linkActivity, loadCatalog, npmGlobalPrefix, planSelfUpdate, readInstalledConsoleVersion, rebuildRetrievalIndex, reimportNutrition, resolveExtractionErrors, resolveAthlete, resolveCatalogPaths, retryDataSync, runSelfUpdateCommand, setAthleteSecret, syncData, syncProgress, connectStrava, connectStravaWithCallback, unlinkActivity, } from '../../runtime/index.js';
 const program = new Command()
     .name('catence-data')
     .description('Athlete-scoped Catence data setup, extraction, and normalization.')
@@ -252,5 +252,30 @@ program.command('build-retrieval-index').description('Build derived local retrie
     const { paths } = await currentAthlete();
     process.stdout.write(`${JSON.stringify(await rebuildRetrievalIndex(paths), null, 2)}\n`);
 });
+program.command('reimport-nutrition').description('Re-run normalization over captured nutrition entities without contacting providers.').action(async () => {
+    const { paths } = await currentAthlete();
+    process.stdout.write(`${JSON.stringify(await reimportNutrition(paths), null, 2)}\n`);
+});
+program
+    .command('import')
+    .description('Import a staged JSONL file (for example a swim-length backfill) into one athlete’s data store.')
+    .requiredOption('--provider <provider>', 'garmin, intervals, or strava')
+    .argument('<file>', 'path to the staged JSONL file')
+    .action(async (file, options) => {
+    const { paths } = await currentAthlete();
+    process.stdout.write(`${JSON.stringify(await importStagedFile(paths, options.provider, file), null, 2)}\n`);
+});
+program
+    .command('resolve-errors')
+    .description('Mark unresolved extraction errors as resolved (for example stale bootstrap failures from an environment that has since recovered).')
+    .option('--run <run-id>', 'resolve errors from a specific run only (repeatable)')
+    .option('--provider <provider>', 'resolve errors for garmin, intervals, or strava only')
+    .option('--before <timestamp>', 'resolve errors created before this ISO timestamp')
+    .option('--all', 'resolve every unresolved error')
+    .action(async (options) => {
+    const { paths } = await currentAthlete();
+    const runIds = Array.isArray(options.run) ? options.run : options.run ? [options.run] : undefined;
+    process.stdout.write(`${JSON.stringify(await resolveExtractionErrors(paths, { runIds, provider: options.provider, before: options.before, all: options.all }), null, 2)}\n`);
+});
 await program.parseAsync();
 //# sourceMappingURL=commands.js.map

package/dist/runtime/index.d.ts

@@ -28,7 +28,7 @@ export { WELLNESS_METRICS, WellnessService } from '../core/query/wellness.js';
 export { jsonSafe, ReadOnlyRepository } from '../core/query/repository.js';
 export { queryReadOnlyData } from '../core/query/sql-guard.js';
 export { createDemoStore, demoStoreMetadata } from '../elt/application/demo.js';
-export { connectStrava, connectStravaWithCallback, dataStatus, disconnectStravaAccount, initializeDataStore, linkActivity, rebuildRetrievalIndex, retryDataSync, syncData, syncProgress, unlinkActivity, } from '../elt/application/management.js';
+export { connectStrava, connectStravaWithCallback, dataStatus, disconnectStravaAccount, initializeDataStore, importStagedFile, linkActivity, rebuildRetrievalIndex, reimportNutrition, resolveExtractionErrors, retryDataSync, syncData, syncProgress, unlinkActivity, } from '../elt/application/management.js';
 export type { ProviderChoice } from '../elt/application/management.js';
 export type { InterruptedRuns, SyncProgressSnapshot, SyncProgressState } from '../contracts/progress.js';
 export { openReadOnlyRepository, ReadOnlyDatabaseError } from '../elt/storage/database.js';

package/dist/runtime/index.js

@@ -15,7 +15,7 @@ export { WELLNESS_METRICS, WellnessService } from '../core/query/wellness.js';
 export { jsonSafe, ReadOnlyRepository } from '../core/query/repository.js';
 export { queryReadOnlyData } from '../core/query/sql-guard.js';
 export { createDemoStore, demoStoreMetadata } from '../elt/application/demo.js';
-export { connectStrava, connectStravaWithCallback, dataStatus, disconnectStravaAccount, initializeDataStore, linkActivity, rebuildRetrievalIndex, retryDataSync, syncData, syncProgress, unlinkActivity, } from '../elt/application/management.js';
+export { connectStrava, connectStravaWithCallback, dataStatus, disconnectStravaAccount, initializeDataStore, importStagedFile, linkActivity, rebuildRetrievalIndex, reimportNutrition, resolveExtractionErrors, retryDataSync, syncData, syncProgress, unlinkActivity, } from '../elt/application/management.js';
 export { openReadOnlyRepository, ReadOnlyDatabaseError } from '../elt/storage/database.js';
 export { DataWriteBusyError } from '../elt/storage/write-lock.js';
 export { hydrateStravaActivity, hydrateStravaSegmentHistory, StravaEnrichmentError, StravaRateLimitError, } from '../elt/ingestion/providers/strava/service.js';

package/docs/deployment/local-mcp.md

@@ -209,6 +209,10 @@ for live runs and only touch the `sync_run_progress` table for recent-run
 history once the lock is free — so they keep working while a backfill holds the
 lock, and never crash with a lock error.
 
+Progress also stays live through the entire run, not just worker extraction: a
+30-second keep-alive covers the JSONL import phase (which emits no worker
+output) and the `intervals` phase of a provider-all sync or backfill.
+
 Other read-only tools (`status`, `query_read_only_data`, and friends) still
 need the database and therefore return a clean `data_sync_in_progress` error
 while a sync run is active. That is expected behavior: the store is locked

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "catence",
-  "version": "0.2.0-beta.11",
+  "version": "0.2.0-beta.13",
   "description": "Local MCP access to normalized Garmin, Intervals.icu, and targeted Strava fitness data.",
   "license": "AGPL-3.0-or-later",
   "repository": {

package/python/catence/providers/garmin/streams.py

@@ -183,6 +183,44 @@ def fit_archive_to_samples(activity_source_id: str, archive: Path) -> list[dict[
         return []
 
 
+def fit_archive_to_swim_lengths(activity_source_id: str, archive: Path) -> list[dict[str, Any]]:
+    """Best-effort per-length parsing from the FIT 'length' messages; malformed/unavailable archives yield no rows."""
+    try:
+        import fitdecode
+        with zipfile.ZipFile(archive) as zip_file:
+            fit_name = next((name for name in zip_file.namelist() if name.lower().endswith(".fit")), None)
+            if not fit_name:
+                return []
+            with zip_file.open(fit_name) as stream, fitdecode.FitReader(stream) as reader:
+                rows: list[dict[str, Any]] = []
+                for frame in reader:
+                    if not isinstance(frame, fitdecode.records.FitDataMessage) or frame.name != "length":
+                        continue
+                    fields = {field.name: field.value for field in frame.fields}
+                    length_type = str(fields.get("length_type") or "").lower()
+                    start_time = fields.get("start_time") or fields.get("timestamp")
+                    row = {
+                        "lengthIndex": _number(fields.get("message_index")),
+                        "lapIndex": None,
+                        "startTime": start_time.isoformat().replace("+00:00", "Z") if hasattr(start_time, "isoformat") else None,
+                        "durationS": _number(fields.get("total_elapsed_time") or fields.get("total_timer_time")),
+                        "activeDurationS": _number(fields.get("total_timer_time")),
+                        "distanceM": _number(fields.get("distance")),
+                        "strokeCount": _number(fields.get("total_strokes")),
+                        "strokeRate": _number(fields.get("avg_swimming_cadence") or fields.get("avg_cadence")),
+                        "swolf": _number(fields.get("avg_swolf")),
+                        "avgHr": _number(fields.get("avg_heart_rate") or fields.get("average_heart_rate")),
+                        "maxHr": _number(fields.get("max_heart_rate")),
+                        "isRest": length_type in {"rest", "idle", "pause"},
+                        "label": str(fields.get("swim_stroke") or "").lower() or None,
+                        "raw": {name: (value.isoformat().replace("+00:00", "Z") if hasattr(value, "isoformat") else value) for name, value in fields.items()},
+                    }
+                    rows.append(row)
+                return rows
+    except (ImportError, OSError, ValueError, zipfile.BadZipFile):
+        return []
+
+
 def write_parquet(data_dir: Path, activity_id: str, start_date: str, samples: Iterable[dict[str, Any]]) -> tuple[Path, str, int]:
     records = list(samples)
     year, month = (start_date[:4] or "unknown"), (start_date[5:7] or "unknown")

package/python/catence/providers/garmin/swim_backfill.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+from .staging import StagingWriter
+from .streams import fit_archive_to_swim_lengths
+
+SWIM_SPORTS = ("lap_swimming", "swimming", "open_water_swimming", "openwaterswimming")
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Backfill swim lengths from already-archived Garmin activity FIT files.")
+    parser.add_argument("--data-dir", required=True, type=Path)
+    parser.add_argument("--output", required=True, type=Path)
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    database_path = args.data_dir / "catence.duckdb"
+    if not database_path.exists():
+        raise SystemExit(f"Catalog database not found: {database_path}")
+    import duckdb
+
+    con = duckdb.connect(str(database_path), read_only=True)
+    rows = con.execute(
+        """
+        SELECT source.remote_activity_id, cast(activity.started_at_utc AS VARCHAR) AS started_at_utc
+        FROM activity_sources AS source
+        JOIN activities AS activity USING (activity_id)
+        WHERE source.provider = 'garmin'
+          AND lower(coalesce(activity.sport, '')) IN ({placeholders})
+        ORDER BY activity.started_at_utc
+        """.format(placeholders=", ".join(f"'{sport}'" for sport in SWIM_SPORTS)),
+    ).fetchall()
+    con.close()
+
+    writer = StagingWriter(args.data_dir, args.output)
+    staged = 0
+    for remote_id, started_at_utc in rows:
+        activity_id = str(remote_id)
+        zip_dir = args.data_dir / "raw" / "garmin" / "activity_original" / activity_id
+        if not zip_dir.is_dir():
+            continue
+        zips = sorted(zip_dir.glob("*.zip"))
+        if not zips:
+            continue
+        archive = zips[-1]
+        lengths = fit_archive_to_swim_lengths(f"garmin:{activity_id}", archive)
+        if not lengths:
+            continue
+        writer.source_entity(
+            "activity_swim_lengths",
+            f"{activity_id}:fit",
+            {"lengths": lengths},
+            archive.stem,
+            parent_remote_id=activity_id,
+            occurred_on=str(started_at_utc)[:10] if started_at_utc else None,
+        )
+        staged += 1
+    print(f"Staged swim lengths for {staged} activities; output: {args.output}")
+
+
+if __name__ == "__main__":
+    main()

package/python/catence/providers/garmin/worker.py

@@ -9,7 +9,7 @@ from typing import Any, Callable, Mapping
 from .progress import ProgressReporter
 from .registry import GARMIN_READ_METHODS, assert_read_only_registry
 from .staging import StagingWriter
-from .streams import SAMPLE_COLUMNS, activity_details_to_samples, activity_power_bests, fit_archive_to_samples, write_parquet
+from .streams import SAMPLE_COLUMNS, activity_details_to_samples, activity_power_bests, fit_archive_to_samples, fit_archive_to_swim_lengths, write_parquet
 
 
 class WorkerInterrupted(Exception):
@@ -556,6 +556,20 @@ class GarminStagingWorker:
                     parent_remote_id=activity_id,
                     occurred_on=start_date[:10],
                 )
+        if original_path and original_path.exists():
+            sport = str(summary.get("sportType") or summary.get("sport") or "").lower()
+            if sport == "lap_swimming" or "swim" in sport:
+                lengths = fit_archive_to_swim_lengths(f"garmin:{activity_id}", original_path)
+                if lengths:
+                    start_date = str(summary.get("startTimeLocal") or summary.get("startTimeGMT") or date.today().isoformat())
+                    self.writer.source_entity(
+                        "activity_swim_lengths",
+                        f"{activity_id}:fit",
+                        {"lengths": lengths},
+                        original_hash,
+                        parent_remote_id=activity_id,
+                        occurred_on=start_date[:10],
+                    )
 
     def _collections(self) -> None:
         today = date.today()