catence 0.2.0-beta.8 → 0.2.0

package/dist/elt/storage/database.js

@@ -2,8 +2,31 @@ import { DuckDBInstance } from '@duckdb/node-api';
 import { randomUUID } from 'node:crypto';
 import { existsSync } from 'node:fs';
 import { ReadOnlyRepository } from '../../core/query/repository.js';
+import { SYNC_STAGES, STALE_RUN_TIMEOUT_MS } from '../../contracts/progress.js';
 import { migrations } from './migrations.js';
 import { json } from './sql.js';
+/**
+ * DuckDB's node API throws "Cannot create values of type ANY" when a bound
+ * parameter resolves to an untyped value. JS `Date` objects surface as `ANY`
+ * (and are not auto-coerced the way strings are), so any date/timestamp
+ * parameter read back from the database and re-bound crashes the statement.
+ * Serializing dates to ISO strings keeps every parameter concretely typed.
+ */
+function serializeValue(value) {
+    if (value instanceof Date)
+        return value.toISOString();
+    return value;
+}
+function serializeBindings(bindings) {
+    if (!bindings)
+        return bindings;
+    if (Array.isArray(bindings))
+        return bindings.map(serializeValue);
+    const out = {};
+    for (const [key, value] of Object.entries(bindings))
+        out[key] = serializeValue(value);
+    return out;
+}
 export class CatenceDatabase {
     instance;
     connection;
@@ -61,10 +84,10 @@ export class CatenceDatabase {
         }
     }
     async run(sql, values) {
-        await this.connection.run(sql, values);
+        await this.connection.run(sql, serializeBindings(values));
     }
     async rows(sql, values) {
-        const result = await this.connection.runAndReadAll(sql, values);
+        const result = await this.connection.runAndReadAll(sql, serializeBindings(values));
         return result.getRowObjectsJS();
     }
     async tableNames(query) {
@@ -81,11 +104,132 @@ export class CatenceDatabase {
         await this.run(`UPDATE sync_runs SET status = CASE WHEN error_count > 0 THEN 'completed_with_errors' ELSE 'completed' END, completed_at = now() WHERE run_id = $runId`, { runId });
         await this.run("UPDATE retrieval_index_state SET status = 'stale' WHERE index_name = 'context'");
     }
+    async heartbeatRun(runId, progress) {
+        await this.run(`INSERT INTO sync_run_progress (run_id, provider, stage, current_step, completed_units, total_units, percent, elapsed_seconds, estimated_remaining_seconds, heartbeat_at, detail_json)
+       VALUES ($runId, $provider, $stage, $currentStep, $completedUnits, $totalUnits, $percent, $elapsedSeconds, $estimatedRemainingSeconds, now(), '{}')
+       ON CONFLICT (run_id) DO UPDATE SET
+         provider = EXCLUDED.provider,
+         stage = EXCLUDED.stage,
+         current_step = EXCLUDED.current_step,
+         completed_units = EXCLUDED.completed_units,
+         total_units = EXCLUDED.total_units,
+         percent = EXCLUDED.percent,
+         elapsed_seconds = EXCLUDED.elapsed_seconds,
+         estimated_remaining_seconds = EXCLUDED.estimated_remaining_seconds,
+         heartbeat_at = now(),
+         detail_json = '{}'`, {
+            runId,
+            provider: progress.provider,
+            stage: progress.stage,
+            currentStep: progress.currentStep,
+            completedUnits: progress.completedUnits,
+            totalUnits: progress.totalUnits,
+            percent: progress.percentComplete,
+            elapsedSeconds: progress.elapsedSeconds,
+            estimatedRemainingSeconds: progress.estimatedRemainingSeconds,
+        });
+    }
+    async markRunInterrupted(runId, stage = 'interrupted') {
+        await this.run("UPDATE sync_runs SET status = 'interrupted', completed_at = now() WHERE run_id = $runId AND status = 'running'", { runId });
+        await this.run('UPDATE sync_run_progress SET stage = $stage, heartbeat_at = now() WHERE run_id = $runId', { runId, stage });
+    }
+    async interruptStaleSyncRuns(timeoutMs = STALE_RUN_TIMEOUT_MS) {
+        const cutoff = new Date(Date.now() - timeoutMs).toISOString();
+        const stale = await this.rows(`SELECT run_id FROM sync_runs WHERE status = 'running' AND started_at < $cutoff`, { cutoff });
+        for (const row of stale) {
+            await this.run("UPDATE sync_runs SET status = 'timed_out', completed_at = now() WHERE run_id = $runId AND status = 'running'", { runId: row.run_id });
+            await this.run("UPDATE sync_run_progress SET stage = 'timed_out', heartbeat_at = now() WHERE run_id = $runId", { runId: row.run_id });
+        }
+        return { runIds: stale.map((row) => row.run_id) };
+    }
+    async syncProgress(timeoutMs = STALE_RUN_TIMEOUT_MS) {
+        // A run killed with its container (docker restart, OOM) keeps
+        // status='running' until something cleans it up, and that cleanup only
+        // runs inside a starting sync child — which the busy guard refuses to
+        // spawn while the row reads as active. Age-filtering here mirrors the
+        // sidecar contract: a heartbeat older than the stale timeout can never
+        // block a new run or disable the dashboard's Sync button forever.
+        const cutoff = new Date(Date.now() - timeoutMs).toISOString();
+        const running = await this.rows(`SELECT runs.run_id, runs.provider,
+         coalesce(progress.stage, 'starting') AS stage,
+         progress.current_step AS current_step,
+         coalesce(progress.completed_units, 0) AS completed_units,
+         progress.total_units AS total_units,
+         coalesce(progress.percent, 0) AS percent,
+         coalesce(progress.elapsed_seconds, 0) AS elapsed_seconds,
+         progress.estimated_remaining_seconds AS estimated_remaining_seconds,
+         cast(coalesce(progress.heartbeat_at, runs.started_at) AS VARCHAR) AS heartbeat_at
+       FROM sync_runs AS runs
+       LEFT JOIN sync_run_progress AS progress USING (run_id)
+       WHERE runs.status = 'running'
+         AND coalesce(progress.heartbeat_at, runs.started_at) > $cutoff
+       ORDER BY runs.started_at DESC`, { cutoff });
+        const recent = await this.rows(`SELECT progress.run_id, progress.provider, progress.stage, progress.current_step,
+         progress.completed_units, progress.total_units, progress.percent, progress.elapsed_seconds,
+         progress.estimated_remaining_seconds, cast(progress.heartbeat_at AS VARCHAR) AS heartbeat_at
+       FROM sync_run_progress AS progress
+       ORDER BY progress.heartbeat_at DESC
+       LIMIT 10`);
+        return { running: running.map(toSyncProgressState), recent: recent.map(toSyncProgressState) };
+    }
+    async hasCalendarEventOlderThan(provider, beforeIsoDate, entityType = 'event') {
+        // Coverage signal for the automatic calendar-events backfill: true once
+        // the store already holds an event older than the regular short window,
+        // meaning the one-time deep pull has landed and normal syncs can stay
+        // narrow. Athletes with genuinely no old events re-run the deep window;
+        // it costs one small paginated listing per sync.
+        const rows = await this.rows(`SELECT 1 FROM source_entities WHERE provider = $provider AND entity_type = $entityType AND occurred_on < $before LIMIT 1`, { provider, entityType, before: beforeIsoDate });
+        return rows.length > 0;
+    }
+    async deleteMissingCalendarEntities(provider, entityTypes, keepRemoteIds, minOccurredOn) {
+        // Calendar entities (joined races, scheduled workouts) are mutable and
+        // deletable upstream. Re-fetches upsert survivors via remote_id; this
+        // removes in-window rows absent from the latest authoritative listing so
+        // upstream deletions propagate instead of leaving ghosts forever. Rows
+        // without a date are preserved (they predate strict dating).
+        if (!entityTypes.length)
+            return;
+        const values = { provider, minOccurredOn };
+        const typePlaceholders = entityTypes.map((name, index) => {
+            values[`type${index}`] = name;
+            return `$type${index}`;
+        });
+        let keepClause = '';
+        if (keepRemoteIds.length) {
+            keepClause = ` AND remote_id NOT IN (${keepRemoteIds.map((_, index) => `$keep${index}`).join(', ')})`;
+            keepRemoteIds.forEach((id, index) => { values[`keep${index}`] = id; });
+        }
+        for (const table of ['source_entities', 'domain_entities']) {
+            await this.run(`DELETE FROM ${table} WHERE provider = $provider AND entity_type IN (${typePlaceholders}) AND occurred_on >= $minOccurredOn${keepClause}`, values);
+        }
+    }
     async addError(runId, provider, endpoint, remoteId, message, retryable) {
         await this.run(`INSERT INTO normalization_errors (error_id, run_id, provider, endpoint, remote_id, message, retryable)
        VALUES ($errorId, $runId, $provider, $endpoint, $remoteId, $message, $retryable)`, { errorId: randomUUID(), runId, provider, endpoint, remoteId, message, retryable });
         await this.run('UPDATE sync_runs SET error_count = error_count + 1 WHERE run_id = $runId', { runId });
     }
+    async resolveErrors(opts = {}) {
+        const clauses = ['resolved_at IS NULL'];
+        const values = {};
+        if (opts.runIds && opts.runIds.length > 0) {
+            const placeholders = opts.runIds.map((_, index) => `$runId${index}`).join(', ');
+            clauses.push(`run_id IN (${placeholders})`);
+            opts.runIds.forEach((runId, index) => { values[`runId${index}`] = runId; });
+        }
+        if (opts.provider) {
+            clauses.push('provider = $provider');
+            values.provider = opts.provider;
+        }
+        if (opts.before) {
+            clauses.push('created_at < $before');
+            values.before = opts.before;
+        }
+        if (!opts.all && clauses.length === 1) {
+            throw new Error('Refusing to resolve all unresolved errors without an explicit --all flag.');
+        }
+        const resolved = await this.rows(`UPDATE normalization_errors SET resolved_at = now() WHERE ${clauses.join(' AND ')} RETURNING error_id`, values);
+        return resolved.length;
+    }
     async status() {
         // @duckdb/node-api owns native statement state on the connection. Keep
         // status reads sequential instead of issuing several native calls at once.
@@ -94,9 +238,11 @@ export class CatenceDatabase {
         const rawObjects = await this.rows('SELECT count(*)::INTEGER AS count FROM raw_objects');
         const activities = await this.rows('SELECT count(*)::INTEGER AS count FROM activities');
         const streams = await this.rows('SELECT count(*)::INTEGER AS count FROM stream_manifest');
+        const errorDetails = await this.rows('SELECT run_id, provider, endpoint, remote_id, message, cast(created_at AS VARCHAR) AS created_at FROM normalization_errors WHERE resolved_at IS NULL ORDER BY created_at DESC LIMIT 100');
         const cursors = await this.rows('SELECT provider, cursor_name, cast(covered_through_date AS VARCHAR) AS covered_through_date, cast(latest_source_date AS VARCHAR) AS latest_source_date, lookback_days, last_successful_run_id, cast(last_completed_at AS VARCHAR) AS last_completed_at, status FROM sync_cursors ORDER BY provider, cursor_name');
         return {
             runs, unresolvedErrors: errors[0]?.count ?? 0, rawObjects: rawObjects[0]?.count ?? 0, activities: activities[0]?.count ?? 0, streams: streams[0]?.count ?? 0,
+            errors: errorDetails,
             cursors: cursors.map((cursor) => ({ ...cursor, next_from_date: subtractDays(cursor.covered_through_date, Number(cursor.lookback_days)) })),
         };
     }
@@ -164,6 +310,24 @@ export class CatenceDatabase {
         const rows = await this.rows('SELECT remote_activity_id, summary_hash FROM activity_sync_state WHERE provider = $provider', { provider });
         return Object.fromEntries(rows.map((row) => [row.remote_activity_id, row.summary_hash]));
     }
+    async hasGarminLactateThresholdHistory() {
+        const rows = await this.rows("SELECT count(*) AS n FROM training_metric_observations WHERE observation_id LIKE 'garmin:lactate_threshold:daily:%'");
+        return (rows[0]?.n ?? 0) > 0;
+    }
+    async earliestActivityStartDate(provider) {
+        const rows = await this.rows(`SELECT min(cast(started_at_utc AS DATE)) AS earliest FROM activities activity
+         JOIN activity_sources source USING (activity_id) WHERE source.provider = $provider`, { provider });
+        return toIsoDate(rows[0]?.earliest ?? null);
+    }
+    async garminSwimActivitySources() {
+        const sports = ['lap_swimming', 'swimming', 'open_water_swimming', 'openwaterswimming'];
+        return this.rows(`SELECT source.remote_activity_id, cast(activity.started_at_utc AS VARCHAR) AS started_on
+         FROM activity_sources AS source
+         JOIN activities AS activity USING (activity_id)
+        WHERE source.provider = 'garmin'
+          AND lower(coalesce(activity.sport, '')) IN (${sports.map((sport) => `'${sport}'`).join(', ')})
+        ORDER BY activity.started_at_utc`);
+    }
     async latestSourceDate(provider, cursorName) {
         const observations = cursorName === 'daily'
             ? `SELECT max(metric_date) AS observed_date FROM daily_metrics WHERE provider = $provider
@@ -281,10 +445,6 @@ export class ReadOnlyCatenceDatabase {
         }
     }
 }
-/** Compose core read services with the DuckDB-backed, read-only adapter. */
-export async function openReadOnlyRepository(paths) {
-    return new ReadOnlyRepository(await ReadOnlyCatenceDatabase.open(paths), paths);
-}
 export class ReadOnlyDatabaseError extends Error {
     code;
     constructor(code, message) {
@@ -293,4 +453,22 @@ export class ReadOnlyDatabaseError extends Error {
         this.name = 'ReadOnlyDatabaseError';
     }
 }
+/** Compose core read services with the DuckDB-backed, read-only adapter. */
+export async function openReadOnlyRepository(paths) {
+    return new ReadOnlyRepository(await ReadOnlyCatenceDatabase.open(paths), paths);
+}
+function toSyncProgressState(row) {
+    return {
+        runId: row.run_id,
+        provider: row.provider,
+        stage: SYNC_STAGES.includes(row.stage) ? row.stage : 'starting',
+        currentStep: row.current_step,
+        completedUnits: Math.max(0, Number(row.completed_units) || 0),
+        totalUnits: row.total_units === null || row.total_units === undefined ? null : Math.max(0, Number(row.total_units) || 0),
+        percentComplete: Math.min(100, Math.max(0, Number(row.percent) || 0)),
+        elapsedSeconds: Math.max(0, Number(row.elapsed_seconds) || 0),
+        estimatedRemainingSeconds: row.estimated_remaining_seconds === null || row.estimated_remaining_seconds === undefined ? null : Math.max(0, Number(row.estimated_remaining_seconds) || 0),
+        heartbeatAt: row.heartbeat_at,
+    };
+}
 //# sourceMappingURL=database.js.map

package/dist/elt/storage/migrations.js

@@ -828,5 +828,149 @@ const swimFactsAndQualityMigration = {
     UPDATE retrieval_index_state SET status = 'stale' WHERE index_name = 'context';
   `,
 };
-export const migrations = [...baseMigrations, stravaAndIdentityMigration, garminPrimaryAndTrainingMetricsMigration, garminActivitySportAndFtpBackfillMigration, garminPrimaryDailyHealthMigration, garminHistoricalMetricsAndStreamsMigration, garminUtcActivityTimestampMigration, garminLactateThresholdMetricsMigration, swimFactsAndQualityMigration];
+const syncRunProgressMigration = {
+    version: 14,
+    name: 'sync_run_progress_heartbeats',
+    sql: `
+    -- Live progress heartbeats for in-flight sync runs. Written by the Node
+    -- side from progress records streamed by provider workers; read-only
+    -- surfaces (status, progress CLI, MCP) join against sync_runs.
+    CREATE TABLE IF NOT EXISTS sync_run_progress (
+      run_id VARCHAR PRIMARY KEY,
+      provider VARCHAR NOT NULL,
+      stage VARCHAR NOT NULL,
+      current_step VARCHAR,
+      completed_units INTEGER NOT NULL DEFAULT 0,
+      total_units INTEGER,
+      percent DOUBLE NOT NULL DEFAULT 0,
+      elapsed_seconds DOUBLE NOT NULL DEFAULT 0,
+      estimated_remaining_seconds DOUBLE,
+      heartbeat_at TIMESTAMPTZ NOT NULL,
+      detail_json JSON NOT NULL DEFAULT '{}'
+    );
+
+    UPDATE retrieval_index_state SET status = 'stale' WHERE index_name = 'context';
+  `,
+};
+const canonicalTrainingDerivedAvgPowerMigration = {
+    version: 15,
+    name: 'canonical_training_derived_avg_power_w',
+    sql: `
+    -- A2: Stop "NULL avg_power => no power" inference without rewriting provenance.
+    -- activity_summaries.avg_power is sparse (NULL for many 2024 running
+    -- activities) while authoritative Garmin FIT-derived power is present as
+    -- per-duration rows in activity_power_bests (exposed as power_best_facts).
+    -- Overwriting the summary value would conflate two layers and break the
+    -- provenance contract (summary vs FIT-derived). Instead, extend the
+    -- canonical view with a derived column that aggregates the FIT signal:
+    -- derived_avg_power_w is the per-activity mean of best_power_w across
+    -- all stored durations. Callers should prefer
+    -- coalesce(avg_power, derived_avg_power_w) or use power_coverage_report /
+    -- read_series on activity_samples for the full signal. NULL remains NULL
+    -- when no FIT bests exist for that activity, making absence explicit.
+    -- Idempotent via CREATE OR REPLACE VIEW; no table mutation.
+    -- Older test fixtures (v9) created activity_summaries with only a subset
+    -- of columns; ensure the full canonical set exists so this view's
+    -- intervals_analysis (which unconditionally references training_load etc.)
+    -- does not throw Binder Error on upgrade.
+    ALTER TABLE activity_summaries ADD COLUMN IF NOT EXISTS elevation_gain_m DOUBLE;
+    ALTER TABLE activity_summaries ADD COLUMN IF NOT EXISTS calories DOUBLE;
+    ALTER TABLE activity_summaries ADD COLUMN IF NOT EXISTS max_hr DOUBLE;
+    ALTER TABLE activity_summaries ADD COLUMN IF NOT EXISTS avg_power DOUBLE;
+    ALTER TABLE activity_summaries ADD COLUMN IF NOT EXISTS weighted_power DOUBLE;
+    ALTER TABLE activity_summaries ADD COLUMN IF NOT EXISTS avg_cadence DOUBLE;
+    ALTER TABLE activity_summaries ADD COLUMN IF NOT EXISTS training_load DOUBLE;
+    ALTER TABLE activity_summaries ADD COLUMN IF NOT EXISTS rpe DOUBLE;
+    ALTER TABLE activity_summaries ADD COLUMN IF NOT EXISTS feel DOUBLE;
+    CREATE OR REPLACE VIEW canonical_activity_training AS
+    WITH ranked AS (
+      SELECT a.*, source.activity_source_id, source.provider, source.remote_activity_id, source.raw_object_hash,
+        summary.*, row_number() OVER (
+          PARTITION BY a.activity_id
+          ORDER BY CASE source.provider WHEN 'garmin' THEN 0 WHEN 'intervals' THEN 1 ELSE 2 END
+        ) AS source_rank
+      FROM activities a
+      JOIN activity_sources source USING (activity_id)
+      LEFT JOIN activity_summaries summary USING (activity_source_id)
+    ), intervals_analysis AS (
+      SELECT source.activity_id,
+        max(summary.training_load) AS intervals_training_load,
+        max(summary.rpe) AS intervals_rpe,
+        max(summary.feel) AS intervals_feel,
+        max(summary.weighted_power) AS intervals_weighted_power
+      FROM activity_sources source
+      JOIN activity_summaries summary USING (activity_source_id)
+      WHERE source.provider = 'intervals'
+      GROUP BY source.activity_id
+    ), derived_power AS (
+      SELECT activity_source_id, avg(best_power_w) AS derived_avg_power_w
+      FROM activity_power_bests
+      GROUP BY activity_source_id
+    )
+    SELECT ranked.* EXCLUDE (source_rank),
+      intervals_analysis.intervals_training_load,
+      intervals_analysis.intervals_rpe,
+      intervals_analysis.intervals_feel,
+      intervals_analysis.intervals_weighted_power,
+      derived_power.derived_avg_power_w
+    FROM ranked
+    LEFT JOIN intervals_analysis USING (activity_id)
+    LEFT JOIN derived_power USING (activity_source_id)
+    WHERE source_rank = 1;
+
+    UPDATE retrieval_index_state SET status = 'stale' WHERE index_name = 'context';
+  `,
+};
+const courseGeometryMigration = {
+    version: 16,
+    name: 'course_geometry',
+    sql: `
+    -- A4: Garmin course/route geometry so a race's elevation/grade profile can be
+    -- resolved and compared against a prior activity's course. garminconnect
+    -- 0.3.x has no dedicated course method, so courses are staged from the raw
+    -- Connect API (connectapi("course-service/course/{id}"), validated live:
+    -- returns metadata + a geoPoints geometry array) and normalized here.
+    -- courseId is the Garmin course identifier referenced by an event's courseId.
+    CREATE TABLE IF NOT EXISTS course_geometry (
+      course_id VARCHAR NOT NULL,
+      seq INTEGER NOT NULL,
+      lat DOUBLE,
+      lon DOUBLE,
+      altitude_m DOUBLE,
+      distance_m DOUBLE,
+      raw_object_hash VARCHAR,
+      PRIMARY KEY (course_id, seq)
+    );
+
+    UPDATE retrieval_index_state SET status = 'stale' WHERE index_name = 'context';
+  `,
+};
+const activityDecouplingFactsMigration = {
+    version: 17,
+    name: 'activity_decoupling_facts',
+    sql: `
+    -- A3 (persistence): make aerobic decoupling (Pa:Hr) and grade-adjusted pace
+    -- (GAP) queryable historically instead of only inside one activity's raw
+    -- metrics_json. Rows are provider-supplied when the provider payload carries
+    -- them (source_type 'provider'); locally derived values are tagged 'derived'
+    -- with caveats_json. Provider values remain authoritative and are never
+    -- overwritten by derivation.
+    CREATE TABLE IF NOT EXISTS activity_decoupling_facts (
+      provider VARCHAR NOT NULL,
+      activity_source_id VARCHAR NOT NULL,
+      sport VARCHAR,
+      started_at_utc TIMESTAMPTZ,
+      metric VARCHAR NOT NULL,
+      value_number DOUBLE,
+      unit VARCHAR,
+      source_type VARCHAR NOT NULL,
+      caveats_json JSON,
+      raw_object_hash VARCHAR,
+      PRIMARY KEY (provider, activity_source_id, metric)
+    );
+
+    UPDATE retrieval_index_state SET status = 'stale' WHERE index_name = 'context';
+  `,
+};
+export const migrations = [...baseMigrations, stravaAndIdentityMigration, garminPrimaryAndTrainingMetricsMigration, garminActivitySportAndFtpBackfillMigration, garminPrimaryDailyHealthMigration, garminHistoricalMetricsAndStreamsMigration, garminUtcActivityTimestampMigration, garminLactateThresholdMetricsMigration, swimFactsAndQualityMigration, syncRunProgressMigration, canonicalTrainingDerivedAvgPowerMigration, courseGeometryMigration, activityDecouplingFactsMigration];
 //# sourceMappingURL=migrations.js.map

package/dist/elt/storage/progress-sidecar.d.ts

@@ -0,0 +1,61 @@
+import { type SyncProgressSnapshot, type SyncProgressState, type SyncStage } from '../../contracts/progress.js';
+import type { CatencePaths } from '../../contracts/runtime.js';
+import type { SyncLogger } from '../../core/logging.js';
+import type { CatenceDatabase } from './database.js';
+export declare const PROGRESS_SIDECAR_SUFFIX = ".progress.json";
+export declare const PROGRESS_SIDECAR_STALE_MS: number;
+export declare const PROGRESS_KEEPALIVE_MS: number;
+export declare const PROGRESS_PUBLISH_THROTTLE_MS: number;
+export declare function progressSidecarPath(paths: CatencePaths, runId: string): string;
+/**
+ * Persist one progress heartbeat as a sidecar file next to the Garmin staging
+ * manifest. Reading this file needs no DuckDB lock, so `progress` stays live
+ * while a sync run holds the single-writer database lock.
+ */
+export declare function writeProgressSidecar(paths: CatencePaths, runId: string, state: SyncProgressState): Promise<void>;
+export declare function removeProgressSidecar(paths: CatencePaths, runId: string): Promise<void>;
+/**
+ * Scan every persisted sidecar heartbeat for an athlete store. Corrupt or
+ * unreadable files are skipped so a torn write can never break progress reads.
+ */
+export declare function readProgressSidecars(paths: CatencePaths): Promise<SyncProgressState[]>;
+/** Active runs from sidecars: non-terminal stage and a fresh heartbeat. */
+export declare function readRunningProgress(paths: CatencePaths, staleAfterMs?: number): Promise<SyncProgressState[]>;
+/**
+ * Merge database-backed progress with sidecar heartbeats. Sidecars win for a
+ * given run (they are freshest); database rows fill runs that never emitted a
+ * sidecar (for example Intervals runs). Recent history is deduplicated across
+ * both sources and sorted by heartbeat recency.
+ */
+export declare function mergeProgress(dbRunning: SyncProgressState[], dbRecent: SyncProgressState[], sidecars: SyncProgressState[]): SyncProgressSnapshot;
+export interface ProgressPumpOptions {
+    database: CatenceDatabase;
+    paths: CatencePaths;
+    log: SyncLogger;
+}
+/**
+ * Drives progress heartbeats for the whole lifetime of a sync run - including
+ * phases that emit no worker output (the Intervals extractor and the Garmin
+ * JSONL import). `publish` throttles writes to the database and sidecar; a
+ * 30-second keep-alive interval refreshes the last known state so a run never
+ * looks stale while it is still active, even during the lock-holding import.
+ */
+export declare class ProgressPump {
+    private readonly runId;
+    private readonly provider;
+    private readonly options;
+    private lastProgress;
+    private lastHeartbeatAt;
+    private interval;
+    private stageOverride;
+    constructor(runId: string, provider: string, options: ProgressPumpOptions);
+    start(): void;
+    publish(state: SyncProgressState): void;
+    /** Force subsequent keep-alive heartbeats to report the given stage. */
+    setStage(stage: SyncStage): void;
+    /** Transition the run to the JSONL import stage; keep-alives persist it. */
+    setStageImporting(): void;
+    /** Write a terminal heartbeat immediately, bypassing the publish throttle. */
+    publishFinal(state: SyncProgressState): Promise<void>;
+    stop(): void;
+}

package/dist/elt/storage/progress-sidecar.js

@@ -0,0 +1,200 @@
+import { existsSync } from 'node:fs';
+import { mkdir, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { normalizeProgress, } from '../../contracts/progress.js';
+export const PROGRESS_SIDECAR_SUFFIX = '.progress.json';
+export const PROGRESS_SIDECAR_STALE_MS = 5 * 60 * 1000;
+export const PROGRESS_KEEPALIVE_MS = 30 * 1000;
+export const PROGRESS_PUBLISH_THROTTLE_MS = 2 * 1000;
+const TERMINAL_STAGES = new Set(['completed', 'failed', 'interrupted', 'timed_out']);
+export function progressSidecarPath(paths, runId) {
+    return path.join(paths.staging, 'garmin', `${runId}${PROGRESS_SIDECAR_SUFFIX}`);
+}
+/**
+ * Persist one progress heartbeat as a sidecar file next to the Garmin staging
+ * manifest. Reading this file needs no DuckDB lock, so `progress` stays live
+ * while a sync run holds the single-writer database lock.
+ */
+export async function writeProgressSidecar(paths, runId, state) {
+    const target = progressSidecarPath(paths, runId);
+    await mkdir(path.dirname(target), { recursive: true });
+    const temporary = `${target}.${Date.now()}.${Math.random().toString(36).slice(2, 10)}.tmp`;
+    await writeFile(temporary, JSON.stringify(state));
+    await rename(temporary, target);
+}
+export async function removeProgressSidecar(paths, runId) {
+    await rm(progressSidecarPath(paths, runId), { force: true });
+}
+/**
+ * Scan every persisted sidecar heartbeat for an athlete store. Corrupt or
+ * unreadable files are skipped so a torn write can never break progress reads.
+ */
+export async function readProgressSidecars(paths) {
+    const directory = path.join(paths.staging, 'garmin');
+    if (!existsSync(directory))
+        return [];
+    const files = (await readdir(directory)).filter((file) => file.endsWith(PROGRESS_SIDECAR_SUFFIX));
+    const states = [];
+    for (const file of files) {
+        try {
+            const state = normalizeProgress(JSON.parse(await readFile(path.join(directory, file), 'utf8')));
+            if (state)
+                states.push(state);
+        }
+        catch {
+            // Torn or partial sidecar write; it is not authoritative.
+        }
+    }
+    return states;
+}
+/** Active runs from sidecars: non-terminal stage and a fresh heartbeat. */
+export async function readRunningProgress(paths, staleAfterMs = PROGRESS_SIDECAR_STALE_MS) {
+    const now = Date.now();
+    const directory = path.join(paths.staging, 'garmin');
+    if (!existsSync(directory))
+        return [];
+    const files = (await readdir(directory)).filter((file) => file.endsWith(PROGRESS_SIDECAR_SUFFIX));
+    const running = [];
+    const prunable = [];
+    for (const file of files) {
+        let state = null;
+        try {
+            state = normalizeProgress(JSON.parse(await readFile(path.join(directory, file), 'utf8')));
+        }
+        catch {
+            // Torn or partial sidecar write; it is not authoritative.
+        }
+        if (!state)
+            continue;
+        const ageMs = now - Date.parse(state.heartbeatAt);
+        if (TERMINAL_STAGES.has(state.stage) || !Number.isFinite(ageMs) || ageMs > staleAfterMs) {
+            // A terminal heartbeat that survived its own cleanup (a crash between
+            // the final publish and the sidecar removal) must never read as an
+            // active run again, so sweep it instead of leaving it behind forever.
+            prunable.push(file);
+            continue;
+        }
+        running.push(state);
+    }
+    await Promise.all(prunable.map((file) => rm(path.join(directory, file), { force: true }).catch(() => undefined)));
+    return running;
+}
+/**
+ * Merge database-backed progress with sidecar heartbeats. Sidecars win for a
+ * given run (they are freshest); database rows fill runs that never emitted a
+ * sidecar (for example Intervals runs). Recent history is deduplicated across
+ * both sources and sorted by heartbeat recency.
+ */
+export function mergeProgress(dbRunning, dbRecent, sidecars) {
+    const running = new Map();
+    for (const state of sidecars)
+        running.set(state.runId, state);
+    for (const state of dbRunning)
+        if (!running.has(state.runId))
+            running.set(state.runId, state);
+    const recent = new Map();
+    for (const state of dbRecent)
+        recent.set(state.runId, state);
+    for (const state of sidecars)
+        recent.set(state.runId, state);
+    return {
+        running: [...running.values()],
+        recent: [...recent.values()]
+            .sort((a, b) => Date.parse(b.heartbeatAt) - Date.parse(a.heartbeatAt))
+            .slice(0, 10),
+    };
+}
+function baselineState(runId, provider, stage) {
+    return {
+        runId,
+        provider,
+        stage,
+        currentStep: null,
+        completedUnits: 0,
+        totalUnits: null,
+        percentComplete: 0,
+        elapsedSeconds: 0,
+        estimatedRemainingSeconds: null,
+        heartbeatAt: new Date().toISOString(),
+    };
+}
+/**
+ * Drives progress heartbeats for the whole lifetime of a sync run - including
+ * phases that emit no worker output (the Intervals extractor and the Garmin
+ * JSONL import). `publish` throttles writes to the database and sidecar; a
+ * 30-second keep-alive interval refreshes the last known state so a run never
+ * looks stale while it is still active, even during the lock-holding import.
+ */
+export class ProgressPump {
+    runId;
+    provider;
+    options;
+    lastProgress = null;
+    lastHeartbeatAt = 0;
+    interval = null;
+    stageOverride = null;
+    constructor(runId, provider, options) {
+        this.runId = runId;
+        this.provider = provider;
+        this.options = options;
+    }
+    start() {
+        if (this.interval)
+            return;
+        this.interval = setInterval(() => {
+            this.publish(this.lastProgress
+                ? { ...this.lastProgress, heartbeatAt: new Date().toISOString() }
+                : baselineState(this.runId, this.provider, this.stageOverride ?? 'starting'));
+        }, PROGRESS_KEEPALIVE_MS);
+        this.interval.unref();
+    }
+    publish(state) {
+        const now = Date.now();
+        if (this.lastHeartbeatAt !== 0 && now - this.lastHeartbeatAt < PROGRESS_PUBLISH_THROTTLE_MS) {
+            this.lastProgress = state;
+            return;
+        }
+        this.lastHeartbeatAt = now;
+        this.lastProgress = state;
+        const { database, paths, log } = this.options;
+        void database.heartbeatRun(this.runId, state).catch((error) => {
+            log.debug('Progress heartbeat failed', { runId: this.runId, error: error instanceof Error ? error.message : String(error) });
+        });
+        void writeProgressSidecar(paths, this.runId, state).catch((error) => {
+            log.debug('Progress sidecar write failed', { runId: this.runId, error: error instanceof Error ? error.message : String(error) });
+        });
+    }
+    /** Force subsequent keep-alive heartbeats to report the given stage. */
+    setStage(stage) {
+        this.stageOverride = stage;
+        if (this.lastProgress) {
+            this.lastProgress = { ...this.lastProgress, stage };
+            this.publish(this.lastProgress);
+        }
+    }
+    /** Transition the run to the JSONL import stage; keep-alives persist it. */
+    setStageImporting() {
+        this.setStage('importing');
+    }
+    /** Write a terminal heartbeat immediately, bypassing the publish throttle. */
+    async publishFinal(state) {
+        this.lastHeartbeatAt = Date.now();
+        this.lastProgress = state;
+        const { database, paths, log } = this.options;
+        // Both terminal writes complete before the caller removes the sidecar, so
+        // a finished run can never resurrect its own progress file after cleanup.
+        await database.heartbeatRun(this.runId, state).catch((error) => {
+            log.debug('Progress heartbeat failed', { runId: this.runId, error: error instanceof Error ? error.message : String(error) });
+        });
+        await writeProgressSidecar(paths, this.runId, state).catch((error) => {
+            log.debug('Progress sidecar write failed', { runId: this.runId, error: error instanceof Error ? error.message : String(error) });
+        });
+    }
+    stop() {
+        if (this.interval) {
+            clearInterval(this.interval);
+            this.interval = null;
+        }
+    }
+}
+//# sourceMappingURL=progress-sidecar.js.map