gpx-from-gopro 0.7.0 → 0.9.0

package/README.md

@@ -20,7 +20,7 @@ No install needed:
 
 ```sh
 npx gpx-from-gopro <dir|file.mp4> [...] [--out DIR] [--tz HOURS] [--rate HZ] [--cache-dir DIR | --no-cache]
-                                  [--organize DIR] [--yes] [--mode core|ski]
+                                  [--organize DIR] [--yes] [--mode core|ski] [--no-gpx]
                                   [--html] [--png [--width N] [--height N]]
 ```
 
@@ -34,7 +34,9 @@ split for restarts/dropouts). A per-file extraction cache (keyed by size+mtime+r
 run resume without re-extracting. `--rate HZ` downsamples from the native ~18 Hz; `--tz HOURS`
 overrides the longitude-guessed local date. `--mode core|ski` runs each session through
 [`gpx-stabilizer`](https://www.npmjs.com/package/gpx-stabilizer)'s `stabilizeTrack` before writing
-(omit for the raw extraction).
+(omit for the raw extraction). `--no-gpx` skips writing the merged `.gpx` entirely — extraction,
+caching, and (with `--organize`) reorganizing the source videos still happen normally; `--html`/
+`--png` are unaffected either way (both render from the raw extracted points, not the `.gpx` file).
 
 ### `--organize DIR` — reorganize the source videos to match the GPX
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gpx-from-gopro",
-  "version": "0.7.0",
+  "version": "0.9.0",
   "description": "Extract a GoPro video's GPS telemetry as GPX or render-agnostic TrackPoints (points + meta + timezone + UTC anchor).",
   "keywords": [
     "gopro",
@@ -32,7 +32,7 @@
     "directory": "packages/gopro"
   },
   "dependencies": {
-    "gpx-stabilizer": "^0.7.0",
+    "gpx-stabilizer": "^0.9.0",
     "egm96-universal": "^1.1.1",
     "gopro-telemetry": "^1.2.11",
     "gpmf-extract": "^0.3.3",

package/src/gopro-cache.js

@@ -8,15 +8,17 @@ import { basename, join, resolve } from "node:path";
 
 // Cache schema version: bump whenever the cached record's shape or the
 // extraction output changes, so stale records read as a miss and re-extract.
+// 4: `meta` also carries `createdUtc` (moov `mvhd.creation_time`) — regressStartUtc's
+// disambiguating reference (telemetry.js), needed on every extraction so bumped
+// rather than left graceful (an old v3 record's absent `createdUtc` would silently
+// fall back to the far-less-precise file mtime instead, on every file until it
+// happens to change).
 // 3: points carry `cts` (media offset); the record is { meta, points, streams },
 // where `streams` holds every non-GPS GPMF channel (IMU/scene/exposure/…) as raw
 // cts samples and `meta` also carries camera fields (`model`/`firmware`/`serial`/
-// `mediaId`/`highlights`, from the moov udta). These additions are
-// ADDITIVE — an older v3 record without them reads back as `streams: {}` / absent
-// fields (graceful), so while the multi-sensor format is still iterated we clear
-// stale records by hand instead of bumping the version; bump before shipping.
+// `mediaId`/`highlights`, from the moov udta).
 // (v2 was { hasGps, meta, points }, pre-cts.)
-export const CACHE_V = 3;
+export const CACHE_V = 4;
 
 /**
  * Cache-record path for a source file: sidecar `<file>.gpxcache.json` by

package/src/gopro-cli.js

@@ -1,9 +1,15 @@
 #!/usr/bin/env node
 // gpx-from-gopro — extract GoPro GPS into merged GPX, one file per camera per local date.
 //   gpx-from-gopro <dir|file.mp4> [...] [--out DIR] [--tz HOURS] [--rate HZ] [--cache-dir DIR | --no-cache]
-//                                [--organize DIR] [--yes] [--mode core|ski]
+//                                [--organize DIR] [--yes] [--mode core|ski] [--no-gpx]
 //                                [--html] [--png [--width N] [--height N]]
 //
+// - --no-gpx: extract + cache (and, with --organize, reorganize) as normal, but skip writing the
+//   merged .gpx to disk. `--organize`'s own gpx-move step already no-ops cleanly when the source
+//   .gpx was never written (organize.js's executeMove checks existsSync(g.from) first), so the two
+//   flags combine with no special-casing needed. --html/--png are unaffected either way — both
+//   render straight from the RAW extracted points (see below), never from the written .gpx file.
+//
 // - --mode core|ski: runs the merged points through gpx-stabilizer's own `stabilizeTrack` (per
 //   recording session — see the `--out` loop below) before writing, instead of shipping today's
 //   default raw extraction. --html/--png (below) render under the SAME mode when both are given,
@@ -69,10 +75,10 @@ const WITH_VALUE = new Set([
   "height",
   "mode",
 ]);
-const KNOWN_BOOL = new Set(["no-cache", "html", "png", "yes"]);
+const KNOWN_BOOL = new Set(["no-cache", "html", "png", "yes", "no-gpx"]);
 const USAGE =
   "usage: gpx-from-gopro <dir|file.mp4> [...] [--out DIR] [--tz HOURS] [--rate HZ]" +
-  " [--cache-dir DIR | --no-cache] [--organize DIR] [--yes] [--mode core|ski]" +
+  " [--cache-dir DIR | --no-cache] [--organize DIR] [--yes] [--mode core|ski] [--no-gpx]" +
   " [--html] [--png [--width N] [--height N]]";
 const inputs = [];
 const opts = {};
@@ -250,15 +256,20 @@ for (const g of groups) {
       type: null,
     },
   };
-  const path = join(outDir, `${g.name}.gpx`);
-  saveGpx(track, path, { creator: "gpx-from-gopro" });
   const npts = tracks.reduce((s, t) => s + t.segments.reduce((s2, seg) => s2 + seg.length, 0), 0);
   const nseg = tracks.reduce((s, t) => s + t.segments.length, 0);
-  written.push(`${g.name}.gpx (${npts} pts, ${tracks.length} trk, ${nseg} seg)`);
+  const stats = `${npts} pts, ${tracks.length} trk, ${nseg} seg`;
+  if (opts["no-gpx"]) {
+    written.push(`[skipped] ${g.name}: gpx not written (--no-gpx), ${stats}`);
+  } else {
+    const path = join(outDir, `${g.name}.gpx`);
+    saveGpx(track, path, { creator: "gpx-from-gopro" });
+    written.push(`${path} (${stats})`);
+  }
 }
 
 console.log(`\ndone. processed=${ok} skipped=${skipped} failed=${failed}`);
-for (const w of written) console.log(`  -> ${join(outDir, w)}`);
+for (const w of written) console.log(`  -> ${w}`);
 
 // ---- --html / --png: eval visualization of each group's merged track, additive to the .gpx above ----
 // (analyzedLayers/analyzedSvg run core's noise-removal pipeline over the flattened group, under the
@@ -307,7 +318,8 @@ async function promptLine(question) {
 }
 
 if (opts.organize) {
-  const includeGpx = !outExplicit; // --out was explicit -> leave the .gpx where the user put it
+  // --out explicit -> leave the .gpx where the user put it; --no-gpx -> there is no .gpx at all
+  const includeGpx = !outExplicit && !opts["no-gpx"];
   const plan = planMove(entries, { root: opts.organize, outDir, includeGpx });
   if (plan.files.length === 0) {
     console.log("\n--organize: nothing to move (no successfully-extracted videos).");

package/src/gopro.js

@@ -116,6 +116,12 @@ function parseHighlights(buf) {
  * @property {string | null} model     camera model from the firmware (e.g. "HERO5"), or null
  * @property {string | null} serial    camera serial (udta `CAME`, hex) — tells two same-model bodies apart
  * @property {string | null} mediaId   global media id (udta `GUMI`, hex) — links a recording's chapter files
+ * @property {number | null} createdUtc  container creation time (moov `mvhd.creation_time`),
+ *   epoch ms, or null if mp4box didn't report one. Camera-written at record time, so — unlike
+ *   file mtime — unaffected by a later copy/move; still the camera's own clock (may be off by
+ *   a fixed local-vs-UTC labelling error, but never off by YEARS the way an unsynced GPS
+ *   chip's firmware boot-time constant can be). Used as `regressStartUtc`'s disambiguating
+ *   reference (telemetry.js).
  * @property {number[]} highlights     user highlight-tag offsets in ms (udta `HMMT`); [] if none
  */
 
@@ -177,6 +183,7 @@ export async function probeGoproMeta(path) {
       model: goproModel(firmware),
       serial: came ? came.toString("hex") : null,
       mediaId: gumi ? gumi.toString("hex") : null,
+      createdUtc: info?.created instanceof Date ? info.created.getTime() : null,
       highlights: parseHighlights(readUdtaRaw(moov, "HMMT")),
     };
   } finally {

package/src/telemetry.js

@@ -67,25 +67,97 @@ export function recordingStartUtc(points) {
 const MIN_REG_POINTS = 5; // too few points → slope is noise
 const MIN_REG_SPAN_MS = 5000; // <5 s of media offset → extrapolating to 0 is unreliable
 const SLOPE_TOL = 0.05; // |slope−1| must be within this (UTC ms vs media ms ⇒ ≈1)
+const MIN_PAIR_SPAN_MS = 1000; // Theil-Sen: skip near-duplicate-cts pairs (unstable slope, tiny denominator)
+const SYNC_RESIDUAL_TOL_MS = 2000; // after the robust fit, >2s off the line predates real time sync
+const REFERENCE_WINDOW_MS = 30 * 24 * 3600 * 1000; // 30 days either side of `referenceUtc`
 
 /**
- * True recording-start instant (UTC) by **linear regression** of each good-fix
- * sample's UTC `time` against its media offset `cts`: `time ≈ intercept +
- * slope·cts`. The recording starts at `cts = 0`, so the intercept is the
- * wall-clock instant of the first video frame — *before* GPS lock, recovering the
- * pre-lock delay that the first-good-fix anchor ignores. `slope` should be ≈ 1
- * (both axes are milliseconds); a slope far from 1 means the GPS UTC and media
- * clocks disagree, so the extrapolation is not trustworthy.
+ * Robust slope estimate: the MEDIAN of pairwise slopes over a grid-sampled subset
+ * (Theil-Sen) — unlike OLS, a minority of wildly-off points can't drag the median,
+ * because their slopes land at the extremes of the sorted list, not the middle.
+ * O(gridN²) pairs; gridN caps at 200 so even a few thousand points stay cheap.
+ * Returns null if every sampled pair fails the `MIN_PAIR_SPAN_MS` separation.
+ */
+function theilSenSlope(pts) {
+  const n = pts.length;
+  const gridN = Math.min(n, 200);
+  const step = Math.max(1, Math.floor(n / gridN));
+  const idxs = [];
+  for (let i = 0; i < n; i += step) idxs.push(i);
+  const slopes = [];
+  for (let i = 0; i < idxs.length; i++) {
+    for (let j = i + 1; j < idxs.length; j++) {
+      const a = pts[idxs[i]];
+      const b = pts[idxs[j]];
+      const dx = b.cts - a.cts;
+      if (Math.abs(dx) < MIN_PAIR_SPAN_MS) continue;
+      slopes.push((b.time - a.time) / dx);
+    }
+  }
+  if (!slopes.length) return null;
+  slopes.sort((x, y) => x - y);
+  return slopes[Math.floor(slopes.length / 2)];
+}
+
+/**
+ * True recording-start instant (UTC) by **linear regression** of each sample's
+ * UTC `time` against its media offset `cts`: `time ≈ intercept + slope·cts`. The
+ * recording starts at `cts = 0`, so the intercept is the wall-clock instant of
+ * the first video frame — *before* GPS lock, recovering the pre-lock delay that
+ * the first-good-fix anchor ignores. `slope` should be ≈ 1 (both axes are
+ * milliseconds); a slope far from 1 means the GPS UTC and media clocks disagree,
+ * so the extrapolation is not trustworthy.
+ *
+ * Deliberately **not** gated on `fix` or position (unlike {@link firstGoodFix}):
+ * some chips (observed on a HERO10) sync UTC `time` well before — or entirely
+ * independently of — ever reporting a `2d`/`3d` fix, so a `time`+`cts` pair can
+ * already be trustworthy while `fix` stays `"none"` and `lat`/`lon` stay stuck at
+ * this format's (0,0) "no sample yet" sentinel for the WHOLE clip (a real
+ * HERO10 track: 5448 samples, zero ever left (0,0), yet the back four-fifths sync
+ * to a slope-1 line). Position and time sync are independent signals — gating the
+ * regression on either misses a valid clock.
+ *
+ * What position-based gating used to catch, this catches differently: a chip
+ * commonly emits a PRE-SYNC run of samples (a firmware boot-time constant,
+ * unrelated to `cts`) before its real UTC sync kicks in — sometimes as most of
+ * the clip, not a minority (observed: an 80/20 split, junk majority). Junk of
+ * this shape isn't just scattered noise, either: it's often internally
+ * consistent (its OWN slope is also ≈1, e.g. a free-running RTC that just never
+ * got corrected) with the WRONG intercept, so a slope check alone can't tell the
+ * two clusters apart — whichever has more points wins a plain majority vote,
+ * right or wrong. `referenceUtc` (the file's own mtime, or any other rough
+ * "roughly when this was recorded" anchor — accurate to a day is plenty) breaks
+ * the tie: real satellite time and a firmware boot constant are NEVER
+ * confusable at that resolution (a boot constant sits years off), so points
+ * more than `REFERENCE_WINDOW_MS` from it are dropped before anything else runs,
+ * regardless of which cluster is bigger. Without a `referenceUtc`, this can only
+ * fall back to catching a scattered-noise minority via {@link theilSenSlope}'s
+ * pairwise-slope median (immune to a minority, not a majority) — pass one
+ * whenever it's available.
  *
- * @param {import("gpx-stabilizer").TrackPoint[]} points  raw points (need `cts` + `time` + `fix`)
+ * @param {import("gpx-stabilizer").TrackPoint[]} points  raw points (need `cts` + `time`)
+ * @param {{ referenceUtc?: number }} [opts]  `referenceUtc`: epoch ms roughly near
+ *   the true recording time (e.g. file mtime) — disambiguates a majority-junk
+ *   cluster from the real one; omit only when no such anchor exists at all.
  * @returns {{ startUtc: number, slope: number, n: number } | null}  null when too
  *   few points, too short a media span, or `cts` is unavailable
  */
-export function regressStartUtc(points) {
-  const pts = (points ?? []).filter(
-    (p) => p && (p.fix === "3d" || p.fix === "2d") && isFiniteNum(p.time) && isFiniteNum(p.cts),
+export function regressStartUtc(points, { referenceUtc } = {}) {
+  let pts = (points ?? []).filter((p) => p && isFiniteNum(p.time) && isFiniteNum(p.cts));
+  if (isFiniteNum(referenceUtc)) {
+    pts = pts.filter((p) => Math.abs(p.time - referenceUtc) <= REFERENCE_WINDOW_MS);
+  }
+  if (pts.length < MIN_REG_POINTS) return null;
+
+  const robustSlope = theilSenSlope(pts);
+  if (robustSlope == null) return null;
+  const intercepts = pts.map((p) => p.time - robustSlope * p.cts).sort((a, b) => a - b);
+  const robustIntercept = intercepts[Math.floor(intercepts.length / 2)];
+  const survivors = pts.filter(
+    (p) => Math.abs(p.time - (robustIntercept + robustSlope * p.cts)) < SYNC_RESIDUAL_TOL_MS,
   );
-  const n = pts.length;
+
+  const n = survivors.length;
   if (n < MIN_REG_POINTS) return null;
   // Centre x (cts) and y (time) before the least-squares sums: UTC ms (~1.7e12)
   // and cts ms make the raw normal equations lose precision to catastrophic
@@ -95,7 +167,7 @@ export function regressStartUtc(points) {
   let sy = 0;
   let minC = Number.POSITIVE_INFINITY;
   let maxC = Number.NEGATIVE_INFINITY;
-  for (const p of pts) {
+  for (const p of survivors) {
     sx += p.cts;
     sy += p.time;
     if (p.cts < minC) minC = p.cts;
@@ -106,7 +178,7 @@ export function regressStartUtc(points) {
   const my = sy / n;
   let sxx = 0;
   let sxy = 0;
-  for (const p of pts) {
+  for (const p of survivors) {
     const dx = p.cts - mx;
     sxx += dx * dx;
     sxy += dx * (p.time - my);
@@ -122,10 +194,11 @@ export function regressStartUtc(points) {
  * true-start when its slope passes the quality gate; otherwise falls back to the
  * first good fix (the pre-lock delay is then unknown and left in).
  * @param {import("gpx-stabilizer").TrackPoint[]} points  raw points
+ * @param {{ referenceUtc?: number }} [opts]  forwarded to {@link regressStartUtc}
  * @returns {{ startUtc: number | null, confidence: 'gps' | null, verified: boolean, slope: number | null }}
  */
-export function resolveStartUtc(points) {
-  const reg = regressStartUtc(points);
+export function resolveStartUtc(points, opts) {
+  const reg = regressStartUtc(points, opts);
   const verified = reg != null && Math.abs(reg.slope - 1) <= SLOPE_TOL;
   const startUtc = verified ? reg.startUtc : recordingStartUtc(points).startUtc;
   return {
@@ -241,7 +314,20 @@ export async function readGoproTelemetry(path, opts = {}) {
   // {lat,lon,ele,time}, dropping the `fix` + `cts` that good-fix selection and the
   // start regression rely on.
   const timezone = timezoneOfPoints(raw);
-  const clock = resolveStartUtc(raw);
+  // regressStartUtc's disambiguating reference (its own doc): prefer the camera's
+  // own container creation time (meta.createdUtc, moov mvhd — set once at record
+  // time) over file mtime, which drifts whenever the file is later copied/moved
+  // (observed: over a MONTH off on a real clip, well past a plausible window) —
+  // mtime is only a fallback for the rare file whose moov didn't yield one.
+  let referenceUtc = meta.createdUtc;
+  if (referenceUtc == null) {
+    try {
+      referenceUtc = statSync(path).mtimeMs;
+    } catch {
+      /* no reference at all — regressStartUtc falls back to unanchored (minority-only) robustness */
+    }
+  }
+  const clock = resolveStartUtc(raw, { referenceUtc });
 
   // resample runs on cleaned survivors, so it IMPLIES stabilize; an explicit
   // `stabilize: false` alongside `resample` is contradictory.