catence 0.2.0-beta.8 → 0.2.0

package/dist/elt/application/management.js

@@ -1,25 +1,262 @@
-import { execFile } from 'node:child_process';
+import { spawn } from 'node:child_process';
 import { randomUUID } from 'node:crypto';
 import { existsSync } from 'node:fs';
-import { mkdir, writeFile } from 'node:fs/promises';
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { createReadStream } from 'node:fs';
+import { createInterface } from 'node:readline';
+import { EVENT_WINDOW_PAST_DAYS } from '../ingestion/providers/registry.js';
 import { createServer } from 'node:http';
 import path from 'node:path';
-import { promisify } from 'node:util';
 import { fileURLToPath } from 'node:url';
+import { normalizeProgress, STALE_RUN_TIMEOUT_MS } from '../../contracts/progress.js';
 import { buildRetrievalIndex } from '../../core/retrieval/index.js';
 import { defaultFromDate, ensurePaths, requireIntervalsConfig } from '../../core/runtime/configuration.js';
+import { createSyncLogger } from '../../core/logging.js';
 import { athleteProviderEnvironment } from '../../core/runtime/secrets.js';
 import { importJsonl } from '../ingestion/importer.js';
 import { IntervalsExtractor } from '../ingestion/providers/intervals.js';
 import { completeStravaAuthorization, disconnectStrava, getStravaAuthorizationUrl, syncStravaGear } from '../ingestion/providers/strava/service.js';
 import { setManualActivityLink, unlinkActivitySource } from '../normalization/activities/linking.js';
-import { CatenceDatabase } from '../storage/database.js';
+import { importSourceEntity } from '../normalization/normalizers.js';
+import { CatenceDatabase, openReadOnlyRepository, ReadOnlyDatabaseError } from '../storage/database.js';
+import { mergeProgress, ProgressPump, readRunningProgress, removeProgressSidecar } from '../storage/progress-sidecar.js';
 import { withDataWriteLock } from '../storage/write-lock.js';
-const execFileAsync = promisify(execFile);
 function packageRoot() {
     return path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../..');
 }
-async function syncProvider(database, paths, provider, options) {
+// Mirrors python/catence/providers/garmin/worker.py fetch windows: the events
+// listing looks back a year and scheduled workouts start three months ago, so
+// only rows inside those windows are authoritative for deletion sweeps.
+const GARMIN_EVENTS_LOOKBACK_DAYS = 365;
+const GARMIN_SCHEDULED_MONTHS_BACK = 3;
+/**
+ * Delete calendar entities that the latest Garmin listing no longer contains.
+ * The staging manifest is the authority on which ids were just fetched; any
+ * in-window row of the same type missing from it was deleted (or un-enrolled)
+ * upstream and would otherwise linger as a ghost.
+ */
+async function reconcileGarminCalendarEntities(database, stagedManifestPath) {
+    const events = [];
+    const scheduled = [];
+    const file = createReadStream(stagedManifestPath, { encoding: 'utf8' });
+    const lines = createInterface({ input: file, crlfDelay: Infinity });
+    for await (const line of lines) {
+        if (!line.trim())
+            continue;
+        let record;
+        try {
+            record = JSON.parse(line);
+        }
+        catch {
+            continue; // The importer owns malformed-line reporting.
+        }
+        const candidate = record;
+        if (candidate?.kind !== 'source_entity' || candidate.provider !== 'garmin')
+            continue;
+        if (candidate.entityType === 'event' && typeof candidate.remoteId === 'string')
+            events.push(candidate.remoteId);
+        if (candidate.entityType === 'scheduled_workout' && typeof candidate.remoteId === 'string')
+            scheduled.push(candidate.remoteId);
+    }
+    const now = new Date();
+    const eventCutoff = new Date(now.getTime() - GARMIN_EVENTS_LOOKBACK_DAYS * 86_400_000).toISOString().slice(0, 10);
+    const scheduledCutoff = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - GARMIN_SCHEDULED_MONTHS_BACK, 1)).toISOString().slice(0, 10);
+    await database.deleteMissingCalendarEntities('garmin', ['event'], events, eventCutoff);
+    await database.deleteMissingCalendarEntities('garmin', ['scheduled_workout'], scheduled, scheduledCutoff);
+}
+function baseProgress(runId, provider, stage) {
+    return {
+        runId, provider, stage, currentStep: null,
+        completedUnits: 0, totalUnits: null, percentComplete: 0, elapsedSeconds: 0,
+        estimatedRemainingSeconds: null, heartbeatAt: new Date().toISOString(),
+    };
+}
+/**
+ * Run the standalone swim-length backfill without progress streaming.
+ * The backfill reads every archived Garmin FIT (already on disk) and emits
+ * source_entity rows for any swim that actually contains length messages;
+ * importing them is idempotent, so re-parsing already-imported swims is
+ * harmless.
+ */
+function runSwimBackfillProcess(arguments_, environment, log) {
+    return new Promise((resolve) => {
+        const child = spawn('uv', arguments_, { ...environment, stdio: ['ignore', 'pipe', 'pipe'] });
+        const stdout = [];
+        const stderr = [];
+        child.stdout.on('data', (chunk) => stdout.push(chunk.toString('utf8')));
+        child.stderr.on('data', (chunk) => stderr.push(chunk.toString('utf8')));
+        child.on('error', (error) => {
+            log.error('Garmin swim-backfill process could not be started', { error: error.message });
+            resolve({ code: 1, signal: null });
+        });
+        child.on('close', (code, signal) => {
+            const out = stdout.join('').trim();
+            const err = stderr.join('').trim();
+            if (out)
+                log.info(`[garmin swim_backfill] ${out}`);
+            if (err)
+                log.error(`[garmin swim_backfill] ${err}`);
+            resolve({ code, signal });
+        });
+    });
+}
+async function backfillGarminSwimLengths(database, paths, parentRunId, log) {
+    const output = path.join(paths.staging, 'garmin', `${parentRunId}.swim_lengths.jsonl`);
+    await mkdir(path.dirname(output), { recursive: true });
+    // Fresh file for deterministic counting; a stale file would make
+    // line counting ambiguous.
+    try {
+        const { rm } = await import('node:fs/promises');
+        await rm(output, { force: true });
+    }
+    catch {
+        // Best-effort cleanup.
+    }
+    // The swim list is queried here and handed to the Python worker as a file:
+    // the child must never open DuckDB itself because the parent holds the
+    // write lock for the whole run.
+    const activitiesFile = path.join(paths.staging, 'garmin', `${parentRunId}.swim_activities.json`);
+    await writeFile(activitiesFile, JSON.stringify(await database.garminSwimActivitySources()));
+    const arguments_ = [
+        'run',
+        'python',
+        '-m',
+        'python.catence.providers.garmin.swim_backfill',
+        '--data-dir',
+        paths.root,
+        '--output',
+        output,
+        '--activities',
+        activitiesFile,
+    ];
+    const environment = {
+        cwd: packageRoot(),
+        env: { ...(await athleteProviderEnvironment(paths)), UV_PROJECT_ENVIRONMENT: process.env.UV_PROJECT_ENVIRONMENT ?? path.join(paths.root, 'python-venv') },
+    };
+    const outcome = await runSwimBackfillProcess(arguments_, environment, log);
+    if (outcome.code !== 0) {
+        throw new Error(`Garmin swim_backfill exited with code ${outcome.code}${outcome.signal ? ` (${outcome.signal})` : ''}.`);
+    }
+    if (!existsSync(output)) {
+        log.info('Garmin swim backfill produced no output file', { parentRunId });
+        return;
+    }
+    const content = await readFile(output, 'utf8');
+    const lines = content.split('\n').filter((line) => line.trim().length > 0);
+    if (lines.length === 0) {
+        log.info('Garmin swim backfill produced no staged lengths', { parentRunId });
+        return;
+    }
+    const backfillRunId = randomUUID();
+    const imported = await importJsonl(database, backfillRunId, output, log);
+    log.info('Imported swim lengths from automatic backfill', { parentRunId, backfillRunId, staged: lines.length, imported });
+}
+/**
+ * Run the Garmin staging worker with streamed stdout/stderr so progress and
+ * errors surface in real time instead of buffering until the child exits.
+ * Progress NDJSON lines (kind 'progress') are normalized and persisted as
+ * throttled heartbeats; other output is forwarded to the sync logger.
+ */
+function runGarminWorker(runId, arguments_, environment, pump, log, onInterruptReady) {
+    return new Promise((resolve) => {
+        const child = spawn('uv', arguments_, { ...environment, stdio: ['ignore', 'pipe', 'pipe'] });
+        let killTimer;
+        onInterruptReady(() => {
+            if (child.exitCode !== null || child.signalCode !== null)
+                return;
+            child.kill('SIGTERM');
+            killTimer = setTimeout(() => {
+                if (child.exitCode !== null || child.signalCode !== null)
+                    return;
+                child.kill('SIGKILL');
+            }, 10_000);
+            killTimer.unref();
+        });
+        const buffer = (stream, sink) => {
+            let pending = '';
+            stream.on('data', (chunk) => {
+                pending += chunk.toString('utf8');
+                let newline;
+                while ((newline = pending.indexOf('\n')) >= 0) {
+                    const line = pending.slice(0, newline).trim();
+                    pending = pending.slice(newline + 1);
+                    if (line)
+                        sink(line);
+                }
+            });
+            stream.on('end', () => {
+                const line = pending.trim();
+                if (line)
+                    sink(line);
+            });
+        };
+        buffer(child.stdout, (line) => {
+            let parsed;
+            try {
+                parsed = JSON.parse(line);
+            }
+            catch {
+                log.info(`[garmin] ${line}`);
+                return;
+            }
+            if (typeof parsed === 'object' && parsed !== null && parsed.kind === 'progress') {
+                const state = normalizeProgress(parsed);
+                if (state)
+                    pump.publish(state);
+                return;
+            }
+            log.info(`[garmin] ${line}`);
+        });
+        buffer(child.stderr, (line) => log.error(`[garmin] ${line}`));
+        child.on('error', (error) => {
+            if (killTimer)
+                clearTimeout(killTimer);
+            resolve({ code: 1, signal: null });
+            log.error('Garmin staging worker could not be started', { runId, error: error.message });
+        });
+        child.on('close', (code, signal) => {
+            if (killTimer)
+                clearTimeout(killTimer);
+            if (code === 0)
+                pump.setStageImporting();
+            resolve({ code, signal });
+        });
+    });
+}
+/**
+ * Trap termination signals for the duration of a sync run so an interrupted
+ * run is marked 'interrupted' in sync_runs instead of being left as a zombie
+ * 'running' row. The process exit code mirrors the received signal.
+ */ async function runWithInterruptGuard(runId, database, onInterrupt, state, work) {
+    let exitCode = 0;
+    const handler = (signal) => {
+        if (state.interrupted)
+            return;
+        state.interrupted = true;
+        exitCode = signal === 'SIGINT' ? 130 : signal === 'SIGTERM' ? 143 : 129;
+        void database.markRunInterrupted(runId).catch(() => undefined);
+        try {
+            onInterrupt();
+        }
+        catch {
+            // The child may not exist yet; the run is still marked interrupted.
+        }
+    };
+    process.once('SIGINT', handler);
+    process.once('SIGTERM', handler);
+    process.once('SIGHUP', handler);
+    try {
+        await work();
+    }
+    finally {
+        process.removeListener('SIGINT', handler);
+        process.removeListener('SIGTERM', handler);
+        process.removeListener('SIGHUP', handler);
+        if (state.interrupted)
+            process.exitCode = exitCode;
+    }
+}
+async function syncProvider(database, paths, provider, options, log) {
     const explicitWindow = options.explicitFrom ? { fromDate: options.explicitFrom, toDate: options.toDate, source: 'explicit' } : null;
     const uncoveredExplicitWindow = async (cursorName) => {
         if (!explicitWindow)
@@ -33,6 +270,7 @@ async function syncProvider(database, paths, provider, options) {
     const activityWindow = explicitWindow ? await uncoveredExplicitWindow('activities') : await database.resolveIncrementalWindow(provider, 'activities', options.toDate, defaultFromDate(), 14);
     const hasProviderWork = provider === 'intervals' ? activityWindow !== null : dailyWindow !== null || activityWindow !== null;
     if (!hasProviderWork) {
+        log.debug('No uncovered work; skipping provider', { provider, daily: dailyWindow ?? null, activities: activityWindow ?? null });
         return {
             runId: null, provider, fromDate: null,
             dailyFromDate: dailyWindow?.fromDate ?? null, dailyToDate: dailyWindow?.toDate ?? null,
@@ -43,13 +281,25 @@ async function syncProvider(database, paths, provider, options) {
     }
     const fromDate = [dailyWindow?.fromDate, activityWindow?.fromDate].filter((date) => Boolean(date)).sort()[0];
     const runId = await database.beginRun(provider, fromDate);
+    log.info('Starting sync run', {
+        runId, provider, fromDate, toDate: options.toDate,
+        dailyFrom: dailyWindow?.fromDate ?? null, dailyTo: dailyWindow?.toDate ?? null,
+        activityFrom: activityWindow?.fromDate ?? null, activityTo: activityWindow?.toDate ?? null,
+        backfill: options.backfill, refreshActivities: options.refreshActivities,
+    });
     let extractionStarted = false;
+    const guard = { interrupted: false };
+    const pump = new ProgressPump(runId, provider, { database, paths, log });
+    pump.start();
     try {
         if (provider === 'intervals') {
             const environment = await athleteProviderEnvironment(paths);
             const config = requireIntervalsConfig(environment);
             extractionStarted = true;
-            await new IntervalsExtractor(database, paths, config.apiKey, config.athleteId).sync(runId, activityWindow, !options.backfill || options.refreshActivities);
+            pump.setStage('intervals');
+            await runWithInterruptGuard(runId, database, () => undefined, guard, async () => {
+                await new IntervalsExtractor(database, paths, config.apiKey, config.athleteId, log).sync(runId, activityWindow, !options.backfill || options.refreshActivities);
+            });
         }
         else {
             const output = path.join(paths.staging, 'garmin', `${runId}.jsonl`);
@@ -71,22 +321,77 @@ async function syncProvider(database, paths, provider, options) {
                 arguments_.push('--skip-activities');
             if (options.backfill && !options.refreshActivities)
                 arguments_.push('--historical-only');
-            await execFileAsync('uv', arguments_, {
-                cwd: packageRoot(),
-                env: { ...(await athleteProviderEnvironment(paths)), UV_PROJECT_ENVIRONMENT: process.env.UV_PROJECT_ENVIRONMENT ?? path.join(paths.root, 'python-venv') },
+            // Data-completeness catch-up: stores synced before lactate-threshold
+            // history capture only hold the latest singleton point. When no
+            // per-day LT observations exist yet, widen the worker's LT window to
+            // the athlete's first activity so the history backfills once; the
+            // condition self-clears after the first successful import.
+            if (!(await database.hasGarminLactateThresholdHistory())) {
+                const earliest = await database.earliestActivityStartDate('garmin');
+                if (earliest)
+                    arguments_.push('--lt-history-from', earliest);
+            }
+            // Calendar-events coverage backfill: a store whose activities predate
+            // the short lookback gets exactly one deep pull (to its earliest
+            // activity); once an older event lands, normal syncs stay narrow.
+            const eventsShortFrom = new Date(Date.now() - EVENT_WINDOW_PAST_DAYS * 86_400_000).toISOString().slice(0, 10);
+            const needsEventBackfill = !(await database.hasCalendarEventOlderThan('garmin', eventsShortFrom));
+            const earliestActivity = needsEventBackfill ? await database.earliestActivityStartDate('garmin') : null;
+            if (needsEventBackfill && earliestActivity && earliestActivity < eventsShortFrom) {
+                arguments_.push('--events-from', earliestActivity);
+            }
+            let interruptChild = null;
+            await runWithInterruptGuard(runId, database, () => interruptChild?.(), guard, async () => {
+                const outcome = await runGarminWorker(runId, arguments_, {
+                    cwd: packageRoot(),
+                    env: { ...(await athleteProviderEnvironment(paths)), UV_PROJECT_ENVIRONMENT: process.env.UV_PROJECT_ENVIRONMENT ?? path.join(paths.root, 'python-venv') },
+                }, pump, log, (kill) => { interruptChild = kill; });
+                if (!guard.interrupted && outcome.code !== 0) {
+                    throw new Error(`Garmin staging worker exited with code ${outcome.code}${outcome.signal ? ` (${outcome.signal})` : ''}.`);
+                }
             });
-            if (!existsSync(output))
-                throw new Error('Garmin staging worker completed without writing a JSONL manifest.');
-            await importJsonl(database, runId, output);
+            if (!guard.interrupted) {
+                if (!existsSync(output))
+                    throw new Error('Garmin staging worker completed without writing a JSONL manifest.');
+                await importJsonl(database, runId, output, log);
+                await reconcileGarminCalendarEntities(database, output);
+                if (provider === 'garmin') {
+                    try {
+                        await backfillGarminSwimLengths(database, paths, runId, log);
+                    }
+                    catch (error) {
+                        log.warn('Garmin swim-length backfill after sync failed', {
+                            runId,
+                            error: error instanceof Error ? error.message : String(error),
+                        });
+                    }
+                }
+            }
         }
     }
     catch (error) {
-        await database.addError(runId, provider, 'sync', null, error instanceof Error ? error.message : String(error), true);
+        if (!guard.interrupted) {
+            const message = error instanceof Error ? error.message : String(error);
+            log.error('Sync run failed', { runId, provider, error: message, stack: error instanceof Error ? error.stack : undefined });
+            await database.addError(runId, provider, 'sync', null, message, true);
+            await database.heartbeatRun(runId, baseProgress(runId, provider, 'failed')).catch(() => undefined);
+        }
     }
     finally {
-        await database.finishRun(runId);
+        pump.stop();
+        if (guard.interrupted) {
+            await database.markRunInterrupted(runId);
+            log.warn('Sync run interrupted', { runId, provider });
+        }
+        else {
+            await database.finishRun(runId);
+            await pump.publishFinal(baseProgress(runId, provider, 'completed'));
+        }
+        await removeProgressSidecar(paths, runId).catch(() => undefined);
+        const [run] = await database.rows('SELECT status, error_count FROM sync_runs WHERE run_id = $runId', { runId });
+        log.info('Sync run finished', { runId, provider, status: run?.status ?? 'unknown', errorCount: run?.error_count ?? 0 });
     }
-    if (options.advanceCursor && !options.explicitFrom && extractionStarted) {
+    if (!guard.interrupted && options.advanceCursor && !options.explicitFrom && extractionStarted) {
         await database.advanceIncrementalCursor(provider, 'daily', runId, options.toDate, 3);
         await database.advanceIncrementalCursor(provider, 'activities', runId, options.toDate, 14);
     }
@@ -109,13 +414,26 @@ export async function syncData(paths, provider, explicitFrom, advanceCursor = tr
     if (provider === 'strava')
         return { provider, result: await syncStravaGear(paths) };
     const database = await CatenceDatabase.open(paths);
+    const log = createSyncLogger(paths);
     try {
+        const timedOutRuns = await database.interruptStaleSyncRuns(STALE_RUN_TIMEOUT_MS);
+        if (timedOutRuns.runIds.length > 0) {
+            log.warn('Marked stale sync runs as timed out', { runIds: timedOutRuns.runIds, timeoutMs: STALE_RUN_TIMEOUT_MS });
+        }
+        for (const runId of timedOutRuns.runIds) {
+            await removeProgressSidecar(paths, runId).catch(() => undefined);
+        }
         const providers = provider === 'all' ? ['intervals', 'garmin'] : [provider];
         const toDate = new Date().toISOString().slice(0, 10);
         const runs = [];
         for (const source of providers)
-            runs.push(await syncProvider(database, paths, source, { explicitFrom, toDate, advanceCursor, refreshActivities, backfill }));
-        return { runIds: runs.flatMap((run) => run.runId ? [run.runId] : []), runs };
+            runs.push(await syncProvider(database, paths, source, { explicitFrom, toDate, advanceCursor, refreshActivities, backfill }, log));
+        const executedRuns = runs.filter((run) => !run.skipped);
+        if (executedRuns.length > 0) {
+            const index = await buildRetrievalIndex(database);
+            log.info('Rebuilt retrieval index after sync', { documents: index.documents, mode: index.mode, watermark: index.watermark });
+        }
+        return { runIds: runs.flatMap((run) => run.runId ? [run.runId] : []), runs, timedOutRuns: timedOutRuns.runIds };
     }
     finally {
         await database.close();
@@ -125,12 +443,14 @@ export async function retryDataSync(paths, previousRunId) {
     await ensurePaths(paths);
     const database = await CatenceDatabase.open(paths);
     try {
+        await database.interruptStaleSyncRuns(STALE_RUN_TIMEOUT_MS);
         const previous = await database.getRun(previousRunId);
         if (!previous)
             throw new Error(`No sync run found for ${previousRunId}.`);
         if (previous.provider === 'strava')
             throw new Error('Retry Strava gear sync with `catence-data sync --provider strava`.');
-        const run = await syncProvider(database, paths, previous.provider, { explicitFrom: previous.from_date, toDate: new Date().toISOString().slice(0, 10), advanceCursor: false, refreshActivities: false, backfill: false });
+        const log = createSyncLogger(paths);
+        const run = await syncProvider(database, paths, previous.provider, { explicitFrom: previous.from_date, toDate: new Date().toISOString().slice(0, 10), advanceCursor: false, refreshActivities: false, backfill: false }, log);
         return { retriedRun: previousRunId, runId: run.runId, run };
     }
     finally {
@@ -271,6 +591,30 @@ export async function dataStatus(paths) {
         await database.close();
     }
 }
+export async function syncProgress(paths) {
+    await ensurePaths(paths);
+    // Live progress must stay readable while a sync run holds the single-writer
+    // DuckDB lock, so the running set comes from sidecar heartbeat files first.
+    const sidecars = await readRunningProgress(paths);
+    let dbRunning = [];
+    let dbRecent = [];
+    try {
+        const repository = await openReadOnlyRepository(paths);
+        try {
+            const snapshot = await repository.progress();
+            dbRunning = snapshot.running;
+            dbRecent = snapshot.recent;
+        }
+        finally {
+            await repository.close();
+        }
+    }
+    catch (error) {
+        if (!(error instanceof ReadOnlyDatabaseError))
+            throw error;
+    }
+    return mergeProgress(dbRunning, dbRecent, sidecars);
+}
 export async function rebuildRetrievalIndex(paths) {
     if (!existsSync(paths.database))
         throw new Error('No Catence database exists yet. Complete a sync before building the retrieval index.');
@@ -282,4 +626,76 @@ export async function rebuildRetrievalIndex(paths) {
         await database.close();
     }
 }
+export async function importStagedFile(paths, provider, filePath) {
+    if (!existsSync(paths.database))
+        throw new Error('No Catence database exists yet. Complete a sync before importing staged records.');
+    if (!existsSync(filePath))
+        throw new Error(`Staging file not found: ${filePath}`);
+    const database = await CatenceDatabase.open(paths);
+    try {
+        const runId = randomUUID();
+        const importedRecords = await importJsonl(database, runId, filePath);
+        return { runId, provider, importedRecords };
+    }
+    finally {
+        await database.close();
+    }
+}
+export async function resolveExtractionErrors(paths, opts = {}) {
+    if (!existsSync(paths.database))
+        throw new Error('No Catence database exists yet.');
+    const database = await CatenceDatabase.open(paths);
+    try {
+        const resolved = await database.resolveErrors({
+            runIds: opts.runIds,
+            provider: opts.provider,
+            before: opts.before,
+            all: opts.all,
+        });
+        return { resolvedErrors: resolved };
+    }
+    finally {
+        await database.close();
+    }
+}
+/**
+ * Re-run normalization over already-captured nutrition entities without
+ * contacting any provider. Idempotent upserts let a fixed normalizer backfill
+ * previously mis-parsed nutrition_days/nutrition_items from stored payloads.
+ */
+export async function reimportNutrition(paths) {
+    if (!existsSync(paths.database))
+        throw new Error('No Catence database exists yet. Complete a sync before re-importing nutrition.');
+    const database = await CatenceDatabase.open(paths);
+    try {
+        const rows = await database.rows(`SELECT provider, entity_type, remote_id, parent_remote_id, occurred_on, source_updated_at, raw_object_hash,
+              cast(payload_json AS VARCHAR) AS payload_json, cast(extension_json AS VARCHAR) AS extension_json
+       FROM source_entities
+       WHERE entity_type IN ('nutrition_log', 'nutrition_day')
+       ORDER BY provider, occurred_on, remote_id`);
+        for (const row of rows) {
+            await importSourceEntity(database, {
+                kind: 'source_entity', schemaVersion: 1, provider: row.provider, entityType: row.entity_type,
+                remoteId: row.remote_id, parentRemoteId: row.parent_remote_id, occurredOn: row.occurred_on, sourceUpdatedAt: row.source_updated_at,
+                rawObjectHash: row.raw_object_hash, payload: parseJsonObject(row.payload_json), extension: parseJsonObject(row.extension_json),
+            });
+        }
+        const days = (await database.rows('SELECT count(*) AS count FROM nutrition_days'))[0]?.count ?? 0;
+        const items = (await database.rows('SELECT count(*) AS count FROM nutrition_items'))[0]?.count ?? 0;
+        return { reimportedEntities: rows.length, nutritionDays: Number(days), nutritionItems: Number(items) };
+    }
+    finally {
+        await database.close();
+    }
+}
+function parseJsonObject(value) {
+    if (!value)
+        return {};
+    try {
+        return JSON.parse(value);
+    }
+    catch {
+        return {};
+    }
+}
 //# sourceMappingURL=management.js.map

package/dist/elt/ingestion/importer.d.ts

@@ -1,7 +1,8 @@
 import type { CatenceDatabase } from '../storage/database.js';
 import { type StagingRecord } from '../../contracts/staging.js';
-export declare function importRecord(database: CatenceDatabase, runId: string, record: StagingRecord): Promise<void>;
-export declare function importJsonl(database: CatenceDatabase, runId: string, filePath: string): Promise<number>;
+import type { SyncLogger } from '../../core/logging.js';
+export declare function importRecord(database: CatenceDatabase, runId: string, record: StagingRecord, log?: SyncLogger): Promise<void>;
+export declare function importJsonl(database: CatenceDatabase, runId: string, filePath: string, log?: SyncLogger): Promise<number>;
 export declare function importSingleStagingFile(database: CatenceDatabase, runId: string, filePath: string): Promise<number>;
 export declare function queueWorkItem(database: CatenceDatabase, runId: string, provider: 'intervals' | 'garmin' | 'strava', endpoint: string, remoteId: string | null, scope: unknown): Promise<string>;
 export declare function completeWorkItem(database: CatenceDatabase, itemId: string): Promise<void>;

package/dist/elt/ingestion/importer.js

@@ -7,7 +7,7 @@ import { importSourceEntity, refreshActivityQuality } from '../normalization/nor
 import { normalizeStravaEntity } from '../normalization/segments/strava-normalizer.js';
 import { reconcileActivityLink } from '../normalization/activities/linking.js';
 import { json } from '../storage/sql.js';
-export async function importRecord(database, runId, record) {
+export async function importRecord(database, runId, record, log) {
     if (record.kind === 'run_manifest') {
         if (record.runId !== runId)
             throw new Error(`Staging manifest run ${record.runId} does not match importer run ${runId}.`);
@@ -33,7 +33,7 @@ export async function importRecord(database, runId, record) {
             if (activity[0]?.activity_id)
                 await refreshActivityQuality(database, activity[0].activity_id);
         }
-        if (record.provider === 'garmin' && ['activity_detail', 'activity_interval', 'activity_exercise_set'].includes(record.entityType) && record.parentRemoteId) {
+        if (record.provider === 'garmin' && ['activity_detail', 'activity_interval', 'activity_exercise_set', 'activity_swim_lengths'].includes(record.entityType) && record.parentRemoteId) {
             const activity = await database.rows('SELECT activity_id FROM activity_sources WHERE activity_source_id = $activitySourceId', { activitySourceId: `garmin:${record.parentRemoteId}` });
             if (activity[0]?.activity_id)
                 await refreshActivityQuality(database, activity[0].activity_id);
@@ -54,8 +54,9 @@ export async function importRecord(database, runId, record) {
         return;
     }
     await database.addError(runId, record.provider, record.endpoint, record.remoteId, record.message, record.retryable);
+    log?.warn('Staging record reported an extraction error', { runId, provider: record.provider, endpoint: record.endpoint, remoteId: record.remoteId, error: record.message });
 }
-export async function importJsonl(database, runId, filePath) {
+export async function importJsonl(database, runId, filePath, log) {
     let count = 0;
     const file = createReadStream(filePath, { encoding: 'utf8' });
     const lines = createInterface({ input: file, crlfDelay: Infinity });
@@ -63,7 +64,7 @@ export async function importJsonl(database, runId, filePath) {
         if (!line.trim())
             continue;
         const record = stagingRecordSchema.parse(JSON.parse(line));
-        await importRecord(database, runId, record);
+        await importRecord(database, runId, record, log);
         count += 1;
     }
     return count;

package/dist/elt/ingestion/providers/intervals.d.ts

@@ -1,16 +1,27 @@
 import type { CatenceDatabase } from '../../storage/database.js';
 import type { CatencePaths } from '../../../contracts/runtime.js';
+import type { SyncLogger } from '../../../core/logging.js';
 export declare class IntervalsExtractor {
     private readonly database;
     private readonly paths;
     private readonly apiKey;
     private readonly athleteId;
+    private readonly log;
     private readonly client;
-    constructor(database: CatenceDatabase, paths: CatencePaths, apiKey: string, athleteId: string);
+    constructor(database: CatenceDatabase, paths: CatencePaths, apiKey: string, athleteId: string, log: SyncLogger);
     sync(runId: string, activityWindow: {
         fromDate: string;
         toDate: string;
     } | null, includeAccount?: boolean): Promise<void>;
+    /**
+     * Regular syncs reach back EVENT_WINDOW_PAST_DAYS — just enough to
+     * associate recently finished events with their activities. A store whose
+     * own activity history predates that span gets an automatic one-time
+     * coverage backfill instead: the window stretches to the earliest
+     * activity until an event older than the short window lands (self-
+     * clearing), after which deeper history is the backfill command's job.
+     */
+    private resolveEventsSyncWindow;
     private captureCollectionChildren;
     private getCollection;
     private captureActivity;