catence 0.2.1-beta.2 → 0.2.1-beta.21

package/dist/elt/application/autosync.d.ts

@@ -0,0 +1,71 @@
+import { z } from 'zod';
+import { type CatalogPaths } from '../../core/runtime/catalog.js';
+import type { CatencePaths } from '../../contracts/runtime.js';
+import { startDetachedSync, type DetachedSyncHandle } from './detached-sync.js';
+/** The sync cadences the scheduler accepts, in minutes. */
+export declare const AUTO_SYNC_INTERVAL_MINUTES: readonly [15, 30, 60, 120, 360, 720, 1440];
+export type AutoSyncIntervalMinutes = (typeof AUTO_SYNC_INTERVAL_MINUTES)[number];
+/** Auto-sync is on by default at 2h: fresh data several times a day, far inside provider limits. */
+export declare const DEFAULT_AUTO_SYNC_INTERVAL_MINUTES: AutoSyncIntervalMinutes;
+declare const autoSyncFileSchema: z.ZodObject<{
+    enabled: z.ZodBoolean;
+    intervalMinutes: z.ZodNumber & z.ZodType<15 | 60 | 30 | 360 | 120 | 720 | 1440, number, z.core.$ZodTypeInternals<15 | 60 | 30 | 360 | 120 | 720 | 1440, number>>;
+    lastTriggeredAt: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+    lastLogFile: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+}, z.core.$strict>;
+export type AutoSyncConfig = z.infer<typeof autoSyncFileSchema>;
+export type AutoSyncSchedule = {
+    athleteId: string;
+    enabled: boolean;
+    intervalMinutes: AutoSyncIntervalMinutes;
+    intervalOptions: ReadonlyArray<{
+        minutes: AutoSyncIntervalMinutes;
+        label: string;
+    }>;
+    lastTriggeredAt: string | null;
+    lastLogFile: string | null;
+    nextRunAt: string | null;
+};
+export declare function autoSyncFilePath(paths: CatencePaths): string;
+/**
+ * Parse one of the interval labels the UI and CLI accept (`15m`, `2h`, `24h`)
+ * or a bare minute count into a valid cadence. Anything else is rejected at
+ * the boundary so stored state can never hold an unsupported interval.
+ */
+export declare function parseAutoSyncInterval(value: string): AutoSyncIntervalMinutes;
+/**
+ * Read the athlete's schedule. A missing file means "on by default": the
+ * effective schedule is reported without writing anything, so a fresh store
+ * stays pristine until the first explicit change (the scheduler does write
+ * trigger bookkeeping once it actually fires).
+ */
+export declare function readAutoSyncSchedule(paths: CatencePaths, athleteId: string): Promise<AutoSyncSchedule>;
+/**
+ * Apply an explicit schedule change (UI or CLI). Only the requested fields
+ * change; trigger bookkeeping survives an interval switch or a pause.
+ */
+export declare function updateAutoSyncSchedule(paths: CatencePaths, athleteId: string, change: {
+    enabled?: boolean;
+    intervalMinutes?: AutoSyncIntervalMinutes;
+}): Promise<AutoSyncSchedule>;
+export type AutoSyncSchedulerOptions = {
+    catalogPaths: CatalogPaths;
+    /** Internal compatibility hook for tests. */
+    startSync?: (request: Parameters<typeof startDetachedSync>[0]) => Promise<DetachedSyncHandle>;
+    now?: () => Date;
+    /** How often due schedules are checked; default every 30 seconds. */
+    tickMs?: number;
+};
+export type AutoSyncScheduler = {
+    stop(): void;
+};
+/**
+ * The auto-sync scheduler: a periodic timer that triggers a detached sync for
+ * every athlete whose schedule is due. It opens no database connection and
+ * takes no locks — the detached child and the single-writer DuckDB lock stay
+ * the only sync executors, so an already-running sync simply makes the tick a
+ * no-op (DetachedSyncBusyError is skipped quietly and retried on a later
+ * tick, once the active run has finished).
+ */
+export declare function startAutoSyncScheduler(options: AutoSyncSchedulerOptions): AutoSyncScheduler;
+export {};

package/dist/elt/application/autosync.js

@@ -0,0 +1,173 @@
+import { mkdir, readFile, rename, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { z } from 'zod';
+import { athleteStorePaths, loadCatalog } from '../../core/runtime/catalog.js';
+import { createSyncLogger } from '../../core/logging.js';
+import { DetachedSyncBusyError, startDetachedSync } from './detached-sync.js';
+/** The sync cadences the scheduler accepts, in minutes. */
+export const AUTO_SYNC_INTERVAL_MINUTES = [15, 30, 60, 120, 360, 720, 1440];
+/** Auto-sync is on by default at 2h: fresh data several times a day, far inside provider limits. */
+export const DEFAULT_AUTO_SYNC_INTERVAL_MINUTES = 120;
+const INTERVAL_CHOICE_LABELS = new Map([
+    [15, '15m'], [30, '30m'], [60, '1h'], [120, '2h'], [360, '6h'], [720, '12h'], [1440, '24h'],
+]);
+const autoSyncFileSchema = z.object({
+    enabled: z.boolean(),
+    intervalMinutes: z.number().int().refine((value) => AUTO_SYNC_INTERVAL_MINUTES.includes(value), 'intervalMinutes must be one of 15, 30, 60, 120, 360, 720, 1440.'),
+    lastTriggeredAt: z.string().datetime().nullable().default(null),
+    lastLogFile: z.string().nullable().default(null),
+}).strict();
+export function autoSyncFilePath(paths) {
+    return path.join(paths.root, 'autosync.json');
+}
+/**
+ * Parse one of the interval labels the UI and CLI accept (`15m`, `2h`, `24h`)
+ * or a bare minute count into a valid cadence. Anything else is rejected at
+ * the boundary so stored state can never hold an unsupported interval.
+ */
+export function parseAutoSyncInterval(value) {
+    const trimmed = value.trim().toLowerCase();
+    const match = /^(\d+)(m|h)$/.exec(trimmed);
+    const minutes = match ? (match[2] === 'h' ? Number(match[1]) * 60 : Number(match[1])) : Number(trimmed);
+    if (Number.isInteger(minutes) && AUTO_SYNC_INTERVAL_MINUTES.includes(minutes)) {
+        return minutes;
+    }
+    throw new Error(`interval must be one of: ${[...INTERVAL_CHOICE_LABELS.values()].join(', ')}.`);
+}
+function defaultConfig() {
+    return { enabled: true, intervalMinutes: DEFAULT_AUTO_SYNC_INTERVAL_MINUTES, lastTriggeredAt: null, lastLogFile: null };
+}
+function nextRunAt(config) {
+    if (!config.enabled || !config.lastTriggeredAt)
+        return null;
+    return new Date(Date.parse(config.lastTriggeredAt) + config.intervalMinutes * 60_000).toISOString();
+}
+function toSchedule(athleteId, config) {
+    return {
+        athleteId,
+        enabled: config.enabled,
+        intervalMinutes: config.intervalMinutes,
+        intervalOptions: AUTO_SYNC_INTERVAL_MINUTES.map((minutes) => ({ minutes, label: INTERVAL_CHOICE_LABELS.get(minutes) ?? `${minutes}m` })),
+        lastTriggeredAt: config.lastTriggeredAt,
+        lastLogFile: config.lastLogFile,
+        nextRunAt: nextRunAt(config),
+    };
+}
+function isMissingFile(error) {
+    return typeof error === 'object' && error !== null && error.code === 'ENOENT';
+}
+async function readEffectiveConfig(paths) {
+    let raw;
+    try {
+        raw = await readFile(autoSyncFilePath(paths), 'utf8');
+    }
+    catch (error) {
+        if (!isMissingFile(error))
+            throw error;
+        return defaultConfig();
+    }
+    const parsed = autoSyncFileSchema.safeParse(JSON.parse(raw));
+    if (!parsed.success) {
+        throw new Error(`autosync.json is not a valid auto-sync schedule (${parsed.error.issues[0]?.message ?? 'unknown issue'}).`);
+    }
+    return parsed.data;
+}
+/**
+ * Read the athlete's schedule. A missing file means "on by default": the
+ * effective schedule is reported without writing anything, so a fresh store
+ * stays pristine until the first explicit change (the scheduler does write
+ * trigger bookkeeping once it actually fires).
+ */
+export async function readAutoSyncSchedule(paths, athleteId) {
+    return toSchedule(athleteId, await readEffectiveConfig(paths));
+}
+async function writeAutoSyncConfig(paths, config) {
+    const file = autoSyncFilePath(paths);
+    // A freshly added athlete has no store directory yet; the schedule file is
+    // often its first artifact.
+    await mkdir(path.dirname(file), { recursive: true });
+    const temporary = `${file}.${process.pid}.tmp`;
+    await writeFile(temporary, `${JSON.stringify(config, null, 2)}\n`, 'utf8');
+    await rename(temporary, file);
+}
+/**
+ * Apply an explicit schedule change (UI or CLI). Only the requested fields
+ * change; trigger bookkeeping survives an interval switch or a pause.
+ */
+export async function updateAutoSyncSchedule(paths, athleteId, change) {
+    const current = await readEffectiveConfig(paths);
+    const updated = {
+        ...current,
+        ...(change.enabled === undefined ? {} : { enabled: change.enabled }),
+        ...(change.intervalMinutes === undefined ? {} : { intervalMinutes: change.intervalMinutes }),
+    };
+    await writeAutoSyncConfig(paths, updated);
+    return toSchedule(athleteId, updated);
+}
+/** Record that the scheduler fired (or attempted to fire) for this athlete. */
+async function recordTrigger(paths, logFile, at) {
+    const config = await readEffectiveConfig(paths);
+    await writeAutoSyncConfig(paths, { ...config, lastTriggeredAt: at.toISOString(), lastLogFile: logFile });
+}
+/**
+ * The auto-sync scheduler: a periodic timer that triggers a detached sync for
+ * every athlete whose schedule is due. It opens no database connection and
+ * takes no locks — the detached child and the single-writer DuckDB lock stay
+ * the only sync executors, so an already-running sync simply makes the tick a
+ * no-op (DetachedSyncBusyError is skipped quietly and retried on a later
+ * tick, once the active run has finished).
+ */
+export function startAutoSyncScheduler(options) {
+    const startSync = options.startSync ?? startDetachedSync;
+    const now = options.now ?? (() => new Date());
+    const tickMs = options.tickMs ?? 30_000;
+    let ticking = false;
+    const tick = async () => {
+        if (ticking)
+            return; // Never overlap ticks; the in-flight one wins.
+        ticking = true;
+        try {
+            const catalog = await loadCatalog(options.catalogPaths);
+            await Promise.allSettled(catalog.athletes.map(async (athlete) => {
+                const paths = athleteStorePaths(options.catalogPaths, athlete.id);
+                const log = createSyncLogger(paths);
+                try {
+                    const config = await readEffectiveConfig(paths);
+                    if (!config.enabled)
+                        return;
+                    const dueAt = config.lastTriggeredAt ? Date.parse(config.lastTriggeredAt) + config.intervalMinutes * 60_000 : 0;
+                    if (now().getTime() < dueAt)
+                        return;
+                    const handle = await startSync({ paths, athleteId: athlete.id, provider: 'all', cliHome: options.catalogPaths.root });
+                    await recordTrigger(paths, handle.logFile, now());
+                    log.info('Auto-sync triggered', { athleteId: athlete.id, intervalMinutes: config.intervalMinutes, logFile: handle.logFile });
+                }
+                catch (error) {
+                    if (error instanceof DetachedSyncBusyError)
+                        return; // A run is active; retry on a later tick.
+                    // Any other failure is recorded as a trigger attempt so a broken
+                    // store cannot be hammered every tick; the next cadence retries.
+                    await recordTrigger(paths, null, now()).catch(() => undefined);
+                    log.error('Auto-sync trigger failed', { athleteId: athlete.id, error: error instanceof Error ? error.message : String(error) });
+                }
+            }));
+        }
+        catch (error) {
+            process.stderr.write(`[catence] Auto-sync tick failed: ${error instanceof Error ? error.message : String(error)}\n`);
+        }
+        finally {
+            ticking = false;
+        }
+    };
+    const timer = setInterval(() => { void tick(); }, tickMs);
+    timer.unref?.();
+    // The first tick is immediate: a restart should heal stale data within a
+    // minute instead of waiting a full interval.
+    void tick();
+    return {
+        stop() {
+            clearInterval(timer);
+        },
+    };
+}
+//# sourceMappingURL=autosync.js.map

package/dist/elt/application/detached-sync.d.ts

@@ -8,6 +8,11 @@ export type DetachedSyncRequest = {
     provider?: DetachedSyncProvider;
     from?: string;
     refresh?: boolean;
+    /**
+     * Run `catence-data backfill` instead of `sync`: extend history backwards
+     * from `from` without advancing the incremental cursors. Requires `from`.
+     */
+    backfill?: boolean;
     /**
      * The `--home` value the detached `catence-data` child must receive.
      *
@@ -35,8 +40,8 @@ export type DetachedSyncSpawner = (command: string, arguments_: readonly string[
     unref(): void;
 };
 /**
- * Spawn one detached `catence-data sync` process for an athlete store and
- * return immediately.
+ * Spawn one detached `catence-data sync` (or `backfill`, when requested)
+ * process for an athlete store and return immediately.
  *
  * The child outlives this process (SSH deaths, Console restarts), logs to a
  * file under the catalog home, and is guarded by the same live-progress

package/dist/elt/application/detached-sync.js

@@ -46,10 +46,15 @@ function assertValidRequest(request) {
     if (request.from !== undefined && !/^\d{4}-\d{2}-\d{2}$/.test(request.from)) {
         throw new Error('from must use YYYY-MM-DD.');
     }
+    // The CLI's backfill subcommand requires --from; enforce it here so the
+    // child never spawns into a usage error.
+    if (request.backfill && !request.from) {
+        throw new Error('from is required for a backfill.');
+    }
 }
 /**
- * Spawn one detached `catence-data sync` process for an athlete store and
- * return immediately.
+ * Spawn one detached `catence-data sync` (or `backfill`, when requested)
+ * process for an athlete store and return immediately.
  *
  * The child outlives this process (SSH deaths, Console restarts), logs to a
  * file under the catalog home, and is guarded by the same live-progress
@@ -73,8 +78,9 @@ export async function startDetachedSync(request, dependencies = {}) {
     const stamp = (dependencies.now ?? (() => new Date()))().toISOString().replace(/[:.]/g, '-');
     const logFile = path.join(request.paths.root, 'logs', `sync-${request.athleteId}-${stamp}.log`);
     await mkdir(path.dirname(logFile), { recursive: true });
-    await writeFile(logFile, `[${new Date().toISOString()}] catence-data --athlete ${request.athleteId} sync --provider ${provider}${request.from ? ` --from ${request.from}` : ''}${request.refresh ? ' --refresh' : ''}\n`, 'utf8');
-    const arguments_ = [cliInvocation().entry, '--home', request.cliHome ?? request.paths.root, '--athlete', request.athleteId, 'sync', '--provider', provider];
+    const subcommand = request.backfill ? 'backfill' : 'sync';
+    await writeFile(logFile, `[${new Date().toISOString()}] catence-data --athlete ${request.athleteId} ${subcommand} --provider ${provider}${request.from ? ` --from ${request.from}` : ''}${request.refresh ? ' --refresh' : ''}\n`, 'utf8');
+    const arguments_ = [cliInvocation().entry, '--home', request.cliHome ?? request.paths.root, '--athlete', request.athleteId, subcommand, '--provider', provider];
     if (request.from)
         arguments_.push('--from', request.from);
     if (request.refresh)

package/dist/elt/application/management.d.ts

@@ -37,4 +37,11 @@ export declare function resolveExtractionErrors(paths: CatencePaths, opts?: {
  * previously mis-parsed nutrition_days/nutrition_items from stored payloads.
  */
 export declare function reimportNutrition(paths: CatencePaths): Promise<Record<string, unknown>>;
+/**
+ * Re-run normalization over already-captured Garmin weigh-in entities without
+ * contacting any provider. Stores synced before beta.5 staged weigh_ins
+ * payloads but never projected them to daily_metrics, so this backfills
+ * weight_kg from stored payloads. Idempotent upserts make it safe to re-run.
+ */
+export declare function reimportWeight(paths: CatencePaths): Promise<Record<string, unknown>>;
 export {};

package/dist/elt/application/management.js

@@ -19,7 +19,7 @@ import { completeStravaAuthorization, disconnectStrava, getStravaAuthorizationUr
 import { setManualActivityLink, unlinkActivitySource } from '../normalization/activities/linking.js';
 import { importSourceEntity } from '../normalization/normalizers.js';
 import { CatenceDatabase, openReadOnlyRepository, ReadOnlyDatabaseError } from '../storage/database.js';
-import { mergeProgress, ProgressPump, readRunningProgress, removeProgressSidecar } from '../storage/progress-sidecar.js';
+import { mergeProgress, ProgressPump, readRunningProgress, removeProgressSidecar, writeProgressSidecar, PROGRESS_KEEPALIVE_MS } from '../storage/progress-sidecar.js';
 import { withDataWriteLock } from '../storage/write-lock.js';
 function packageRoot() {
     return path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../..');
@@ -114,9 +114,16 @@ async function backfillGarminSwimLengths(database, paths, parentRunId, log) {
     }
     // The swim list is queried here and handed to the Python worker as a file:
     // the child must never open DuckDB itself because the parent holds the
-    // write lock for the whole run.
+    // write lock for the whole run. Only activities without per-length rows are
+    // handed over: re-parsing every archived FIT on every sync cost ~23s/run
+    // with nothing new to stage.
     const activitiesFile = path.join(paths.staging, 'garmin', `${parentRunId}.swim_activities.json`);
-    await writeFile(activitiesFile, JSON.stringify(await database.garminSwimActivitySources()));
+    const pendingSwims = await database.garminSwimActivitySourcesWithoutLengths();
+    if (pendingSwims.length === 0) {
+        log.info('Garmin swim backfill skipped: every swim activity already has lengths', { parentRunId });
+        return;
+    }
+    await writeFile(activitiesFile, JSON.stringify(pendingSwims));
     const arguments_ = [
         'run',
         'python',
@@ -207,7 +214,21 @@ function runGarminWorker(runId, arguments_, environment, pump, log, onInterruptR
             }
             log.info(`[garmin] ${line}`);
         });
-        buffer(child.stderr, (line) => log.error(`[garmin] ${line}`));
+        buffer(child.stderr, (line) => {
+            let parsed;
+            try {
+                parsed = JSON.parse(line);
+            }
+            catch {
+                log.error(`[garmin] ${line}`);
+                return;
+            }
+            if (typeof parsed === 'object' && parsed !== null && parsed.kind === 'stage_timings') {
+                log.info('Garmin staging worker stage timings', parsed);
+                return;
+            }
+            log.error(`[garmin] ${line}`);
+        });
         child.on('error', (error) => {
             if (killTimer)
                 clearTimeout(killTimer);
@@ -310,7 +331,9 @@ async function syncProvider(database, paths, provider, options, log) {
             // streams even when the list-summary hash is unchanged.
             await writeFile(knownActivities, JSON.stringify(options.refreshActivities ? {} : await database.knownActivitySummaryHashes('garmin')));
             extractionStarted = true;
-            const arguments_ = ['run', 'python', '-m', 'python.catence.providers.garmin.cli', '--from', fromDate, '--to', options.toDate, '--data-dir', paths.root, '--output', output, '--run-id', runId, '--known-activities', knownActivities];
+            const requestedConcurrency = Number.parseInt(process.env.CATENCE_GARMIN_CONCURRENCY ?? '5', 10);
+            const workerConcurrency = Number.isFinite(requestedConcurrency) && requestedConcurrency > 0 ? Math.trunc(requestedConcurrency) : 5;
+            const arguments_ = ['run', 'python', '-m', 'python.catence.providers.garmin.cli', '--from', fromDate, '--to', options.toDate, '--data-dir', paths.root, '--output', output, '--run-id', runId, '--known-activities', knownActivities, '--concurrency', String(workerConcurrency)];
             if (dailyWindow)
                 arguments_.push('--daily-from', dailyWindow.fromDate, '--daily-to', dailyWindow.toDate);
             else
@@ -340,6 +363,7 @@ async function syncProvider(database, paths, provider, options, log) {
             if (needsEventBackfill && earliestActivity && earliestActivity < eventsShortFrom) {
                 arguments_.push('--events-from', earliestActivity);
             }
+            const workerStartedAt = Date.now();
             let interruptChild = null;
             await runWithInterruptGuard(runId, database, () => interruptChild?.(), guard, async () => {
                 const outcome = await runGarminWorker(runId, arguments_, {
@@ -350,14 +374,21 @@ async function syncProvider(database, paths, provider, options, log) {
                     throw new Error(`Garmin staging worker exited with code ${outcome.code}${outcome.signal ? ` (${outcome.signal})` : ''}.`);
                 }
             });
+            log.info('Garmin staging worker finished', { runId, durationMs: Date.now() - workerStartedAt, concurrency: workerConcurrency });
             if (!guard.interrupted) {
                 if (!existsSync(output))
                     throw new Error('Garmin staging worker completed without writing a JSONL manifest.');
+                let phaseStartedAt = Date.now();
                 await importJsonl(database, runId, output, log);
+                log.info('Garmin staging import finished', { runId, durationMs: Date.now() - phaseStartedAt });
+                phaseStartedAt = Date.now();
                 await reconcileGarminCalendarEntities(database, output);
+                log.info('Garmin calendar reconciliation finished', { runId, durationMs: Date.now() - phaseStartedAt });
                 if (provider === 'garmin') {
                     try {
+                        phaseStartedAt = Date.now();
                         await backfillGarminSwimLengths(database, paths, runId, log);
+                        log.info('Garmin swim-length backfill finished', { runId, durationMs: Date.now() - phaseStartedAt });
                     }
                     catch (error) {
                         log.warn('Garmin swim-length backfill after sync failed', {
@@ -430,8 +461,43 @@ export async function syncData(paths, provider, explicitFrom, advanceCursor = tr
             runs.push(await syncProvider(database, paths, source, { explicitFrom, toDate, advanceCursor, refreshActivities, backfill }, log));
         const executedRuns = runs.filter((run) => !run.skipped);
         if (executedRuns.length > 0) {
-            const index = await buildRetrievalIndex(database);
-            log.info('Rebuilt retrieval index after sync', { documents: index.documents, mode: index.mode, watermark: index.watermark });
+            // The per-provider pumps already stopped (sidecars removed, runs marked
+            // completed), but buildRetrievalIndex still holds the single-writer DB
+            // lock, so /dashboard reads as unavailable. Hold a non-terminal sidecar
+            // for the last executed run so /sync/status keeps reporting running and
+            // the Status Sync button stays disabled through local post-processing.
+            const last = [...executedRuns].reverse().find((run) => run.runId);
+            let indexingKeepalive = null;
+            const holdIndexingProgress = () => last?.runId
+                ? writeProgressSidecar(paths, last.runId, {
+                    ...baseProgress(last.runId, last.provider, 'importing'),
+                    currentStep: 'building retrieval index',
+                    percentComplete: 99,
+                }).catch((error) => {
+                    log.debug('Indexing progress sidecar write failed', {
+                        runId: last.runId,
+                        error: error instanceof Error ? error.message : String(error),
+                    });
+                })
+                : Promise.resolve();
+            if (last?.runId) {
+                await holdIndexingProgress();
+                indexingKeepalive = setInterval(() => {
+                    void holdIndexingProgress();
+                }, PROGRESS_KEEPALIVE_MS);
+                indexingKeepalive.unref?.();
+            }
+            try {
+                const indexStartedAt = Date.now();
+                const index = await buildRetrievalIndex(database);
+                log.info('Rebuilt retrieval index after sync', { documents: index.documents, mode: index.mode, watermark: index.watermark, durationMs: Date.now() - indexStartedAt });
+            }
+            finally {
+                if (indexingKeepalive)
+                    clearInterval(indexingKeepalive);
+                if (last?.runId)
+                    await removeProgressSidecar(paths, last.runId).catch(() => undefined);
+            }
         }
         return { runIds: runs.flatMap((run) => run.runId ? [run.runId] : []), runs, timedOutRuns: timedOutRuns.runIds };
     }
@@ -688,6 +754,36 @@ export async function reimportNutrition(paths) {
         await database.close();
     }
 }
+/**
+ * Re-run normalization over already-captured Garmin weigh-in entities without
+ * contacting any provider. Stores synced before beta.5 staged weigh_ins
+ * payloads but never projected them to daily_metrics, so this backfills
+ * weight_kg from stored payloads. Idempotent upserts make it safe to re-run.
+ */
+export async function reimportWeight(paths) {
+    if (!existsSync(paths.database))
+        throw new Error('No Catence database exists yet. Complete a sync before re-importing weight.');
+    const database = await CatenceDatabase.open(paths);
+    try {
+        const rows = await database.rows(`SELECT provider, entity_type, remote_id, parent_remote_id, occurred_on, source_updated_at, raw_object_hash,
+              cast(payload_json AS VARCHAR) AS payload_json, cast(extension_json AS VARCHAR) AS extension_json
+       FROM source_entities
+       WHERE provider = 'garmin' AND entity_type = 'body_composition'
+       ORDER BY occurred_on, remote_id`);
+        for (const row of rows) {
+            await importSourceEntity(database, {
+                kind: 'source_entity', schemaVersion: 1, provider: row.provider, entityType: row.entity_type,
+                remoteId: row.remote_id, parentRemoteId: row.parent_remote_id, occurredOn: row.occurred_on, sourceUpdatedAt: row.source_updated_at,
+                rawObjectHash: row.raw_object_hash, payload: parseJsonObject(row.payload_json), extension: parseJsonObject(row.extension_json),
+            });
+        }
+        const weights = (await database.rows(`SELECT count(*) AS count FROM daily_metrics WHERE metric_name = 'weight_kg'`))[0]?.count ?? 0;
+        return { reimportedEntities: rows.length, weightObservations: Number(weights) };
+    }
+    finally {
+        await database.close();
+    }
+}
 function parseJsonObject(value) {
     if (!value)
         return {};

package/dist/elt/ingestion/importer.d.ts

@@ -1,7 +1,17 @@
 import type { CatenceDatabase } from '../storage/database.js';
 import { type StagingRecord } from '../../contracts/staging.js';
 import type { SyncLogger } from '../../core/logging.js';
-export declare function importRecord(database: CatenceDatabase, runId: string, record: StagingRecord, log?: SyncLogger): Promise<void>;
+/**
+ * Per-pass import state. When present, activity-quality refreshes are
+ * deferred to the end of the pass: a staging file lists an activity before
+ * its detail records, so refreshing inline would compute flags from partial
+ * data, and one refresh per activity is enough regardless of how many
+ * detail entity types follow it.
+ */
+export type ImportPassContext = {
+    deferredQualityActivityIds: Set<string>;
+};
+export declare function importRecord(database: CatenceDatabase, runId: string, record: StagingRecord, log?: SyncLogger, pass?: ImportPassContext): Promise<void>;
 export declare function importJsonl(database: CatenceDatabase, runId: string, filePath: string, log?: SyncLogger): Promise<number>;
 export declare function importSingleStagingFile(database: CatenceDatabase, runId: string, filePath: string): Promise<number>;
 export declare function queueWorkItem(database: CatenceDatabase, runId: string, provider: 'intervals' | 'garmin' | 'strava', endpoint: string, remoteId: string | null, scope: unknown): Promise<string>;

package/dist/elt/ingestion/importer.js

@@ -7,7 +7,12 @@ import { importSourceEntity, refreshActivityQuality } from '../normalization/nor
 import { normalizeStravaEntity } from '../normalization/segments/strava-normalizer.js';
 import { reconcileActivityLink } from '../normalization/activities/linking.js';
 import { json } from '../storage/sql.js';
-export async function importRecord(database, runId, record, log) {
+const GARMIN_ACTIVITY_DETAIL_ENTITY_TYPES = ['activity_detail', 'activity_interval', 'activity_exercise_set', 'activity_swim_lengths'];
+async function activityIdForSource(database, activitySourceId) {
+    const activity = await database.rows('SELECT activity_id FROM activity_sources WHERE activity_source_id = $activitySourceId', { activitySourceId });
+    return activity[0]?.activity_id ?? null;
+}
+export async function importRecord(database, runId, record, log, pass) {
     if (record.kind === 'run_manifest') {
         if (record.runId !== runId)
             throw new Error(`Staging manifest run ${record.runId} does not match importer run ${runId}.`);
@@ -29,14 +34,14 @@ export async function importRecord(database, runId, record, log) {
             await importSourceEntity(database, record);
         if (record.entityType === 'activity') {
             await reconcileActivityLink(database, record.provider + ':' + record.remoteId);
-            const activity = await database.rows('SELECT activity_id FROM activity_sources WHERE activity_source_id = $activitySourceId', { activitySourceId: record.provider + ':' + record.remoteId });
-            if (activity[0]?.activity_id)
-                await refreshActivityQuality(database, activity[0].activity_id);
+            const activityId = await activityIdForSource(database, record.provider + ':' + record.remoteId);
+            if (activityId)
+                await queueActivityQualityRefresh(database, activityId, pass);
         }
-        if (record.provider === 'garmin' && ['activity_detail', 'activity_interval', 'activity_exercise_set', 'activity_swim_lengths'].includes(record.entityType) && record.parentRemoteId) {
-            const activity = await database.rows('SELECT activity_id FROM activity_sources WHERE activity_source_id = $activitySourceId', { activitySourceId: `garmin:${record.parentRemoteId}` });
-            if (activity[0]?.activity_id)
-                await refreshActivityQuality(database, activity[0].activity_id);
+        if (record.provider === 'garmin' && GARMIN_ACTIVITY_DETAIL_ENTITY_TYPES.includes(record.entityType) && record.parentRemoteId) {
+            const activityId = await activityIdForSource(database, `garmin:${record.parentRemoteId}`);
+            if (activityId)
+                await queueActivityQualityRefresh(database, activityId, pass);
         }
         return;
     }
@@ -56,17 +61,61 @@ export async function importRecord(database, runId, record, log) {
     await database.addError(runId, record.provider, record.endpoint, record.remoteId, record.message, record.retryable);
     log?.warn('Staging record reported an extraction error', { runId, provider: record.provider, endpoint: record.endpoint, remoteId: record.remoteId, error: record.message });
 }
+/**
+ * Refresh now (single-record callers) or defer to the end of the pass
+ * (bulk importJsonl callers) — see ImportPassContext.
+ */
+async function queueActivityQualityRefresh(database, activityId, pass) {
+    if (pass) {
+        pass.deferredQualityActivityIds.add(activityId);
+        return;
+    }
+    await refreshActivityQuality(database, activityId);
+}
+/** Records per transaction: large enough to amortize DuckDB commits, small enough to bound a mid-file failure. */
+const IMPORT_CHUNK_SIZE = 200;
 export async function importJsonl(database, runId, filePath, log) {
     let count = 0;
+    const pass = { deferredQualityActivityIds: new Set() };
     const file = createReadStream(filePath, { encoding: 'utf8' });
     const lines = createInterface({ input: file, crlfDelay: Infinity });
-    for await (const line of lines) {
-        if (!line.trim())
-            continue;
-        const record = stagingRecordSchema.parse(JSON.parse(line));
-        await importRecord(database, runId, record, log);
-        count += 1;
+    let inTransaction = false;
+    const startedAt = Date.now();
+    try {
+        for await (const line of lines) {
+            if (!line.trim())
+                continue;
+            if (count % IMPORT_CHUNK_SIZE === 0) {
+                await database.run('BEGIN TRANSACTION');
+                inTransaction = true;
+            }
+            const record = stagingRecordSchema.parse(JSON.parse(line));
+            await importRecord(database, runId, record, log, pass);
+            count += 1;
+            if (count % IMPORT_CHUNK_SIZE === 0) {
+                await database.run('COMMIT');
+                inTransaction = false;
+            }
+        }
+    }
+    catch (error) {
+        if (inTransaction)
+            await database.run('ROLLBACK');
+        throw error;
+    }
+    if (inTransaction) {
+        await database.run('COMMIT');
+        inTransaction = false;
+    }
+    const qualityStartedAt = Date.now();
+    for (const activityId of pass.deferredQualityActivityIds) {
+        await refreshActivityQuality(database, activityId);
     }
+    log?.info('Staging import finished with stage timings', {
+        runId, records: count, durationMs: Date.now() - startedAt,
+        qualityRefreshes: pass.deferredQualityActivityIds.size,
+        qualityRefreshMs: Date.now() - qualityStartedAt,
+    });
     return count;
 }
 export async function importSingleStagingFile(database, runId, filePath) {

package/dist/elt/ingestion/providers/intervals.d.ts

@@ -27,6 +27,13 @@ export declare class IntervalsExtractor {
     private captureActivity;
     private capture;
     private importEntities;
+    /**
+     * Intervals.icu enforces 2500 requests per rolling 15 minutes and 5000 per
+     * day; over-limit calls answer 429 with a Retry-After header. Back off
+     * cooperatively and retry a bounded number of times so one auto-sync run
+     * degrades instead of recording a wall of retryable errors.
+     */
+    private request;
     private getJson;
     private getBinary;
 }