@remnic/core 9.64.0 → 9.64.1

package/src/location/pipeline.ts

@@ -0,0 +1,245 @@
+/**
+ * Location day-sync pipeline (issue #2044).
+ *
+ * For one local day, each configured (enabled, registered) provider's
+ * observations are fetched over the day's half-open UTC window, folded into
+ * place-visit segments, merged with the other sources' last-known day
+ * payloads from sync state, and rendered to the day document. Sync state
+ * advances only AFTER the day document (when any) is durable, so a failed
+ * write never moves the watermark past data that did not land. An empty
+ * fetch is a successful sync — never conflated with a provider failure
+ * (AGENTS.md rule 22).
+ */
+
+import { getLocationProvider } from "./registry.js";
+import { locationDayWindow, observationSegments } from "./intervals.js";
+import {
+  composeLocationDayBody,
+  composeLocationDayMeta,
+  loadLocationSyncState,
+  serializeLocationDay,
+  updateLocationSourceDay,
+  writeLocationDay,
+  type LocationDaySourceEntry,
+} from "./store.js";
+import type { LocationConfig, LocationObservation, LocationProvider, LocationSegment } from "./types.js";
+
+const MAX_SYNC_PAGES = 10_000;
+
+export interface LocationDaySyncOptions {
+  config: LocationConfig;
+  memoryDir: string;
+  date: string;
+  signal?: AbortSignal;
+  /** Runaway-pagination guard; defaults to MAX_SYNC_PAGES. */
+  maxPages?: number;
+  now?: () => Date;
+  /** Provider lookup seam (defaults to the registry); tests inject doubles. */
+  getProvider?: (id: string) => LocationProvider | undefined;
+}
+
+export interface LocationSourceSyncResult {
+  sourceId: string;
+  status: "synced" | "skipped" | "failed";
+  skipReason?: "source-disabled" | "provider-not-registered";
+  fetched: number;
+  dayWritten: boolean;
+  stateSaved: boolean;
+  /** Sanitized provider error (no tokens, URLs, or coordinates). */
+  error?: string;
+}
+
+/** Redact credential, URL query, and coordinate shapes from a provider error. */
+export function sanitizeProviderError(error: unknown): string {
+  const raw = error instanceof Error ? error.message : String(error);
+  return raw
+    .replace(/Bearer\s+[A-Za-z0-9._~+/=-]+/gi, "Bearer [redacted]")
+    .replace(/[?#][^ \t"']*/g, "[redacted]")
+    .replace(/[+-]?\d+\.\d{3,},\s?[+-]?\d+\.\d{3,}/g, "[coordinates]")
+    .slice(0, 300);
+}
+
+function assertValidObservation(observation: LocationObservation): void {
+  if (typeof observation !== "object" || observation === null) {
+    throw new TypeError("location provider returned a non-object observation");
+  }
+  if (typeof observation.observedAtUtc !== "string" || !Number.isFinite(Date.parse(observation.observedAtUtc))) {
+    throw new TypeError("location provider returned an observation with an invalid observedAtUtc");
+  }
+  const place = observation.place;
+  if (typeof place !== "object" || place === null || typeof place.id !== "string" || place.id.length === 0) {
+    throw new TypeError("location provider returned an observation without a valid place");
+  }
+  if (typeof place.label !== "string" || place.label.trim().length === 0) {
+    throw new TypeError(`location provider returned place '${place.id}' without a label`);
+  }
+}
+
+/** Fetch every observation one provider holds for the window, all pages. */
+async function fetchWindow(
+  provider: LocationProvider,
+  window: { startUtc: string; endUtc: string },
+  options: LocationDaySyncOptions,
+): Promise<LocationObservation[]> {
+  const maxPages = options.maxPages ?? MAX_SYNC_PAGES;
+  if (!Number.isInteger(maxPages) || maxPages < 1) {
+    throw new RangeError("location sync maxPages must be a positive integer");
+  }
+  const observations: LocationObservation[] = [];
+  const seenCursors = new Set<string>();
+  let cursor: string | null = null;
+  let pages = 0;
+  let completed = false;
+  while (pages < maxPages) {
+    options.signal?.throwIfAborted();
+    const page = await provider.fetchObservations({
+      startUtc: window.startUtc,
+      endUtc: window.endUtc,
+      cursor,
+      signal: options.signal,
+    });
+    pages += 1;
+    for (const observation of page.observations) assertValidObservation(observation);
+    observations.push(...page.observations);
+    if (page.nextCursor === null) {
+      completed = true;
+      break;
+    }
+    if (typeof page.nextCursor !== "string" || page.nextCursor.length === 0) {
+      throw new TypeError("location provider returned an invalid cursor");
+    }
+    if (seenCursors.has(page.nextCursor)) {
+      throw new Error("location provider returned a repeated cursor");
+    }
+    seenCursors.add(page.nextCursor);
+    cursor = page.nextCursor;
+  }
+  // Only a loop that ran out of its page budget without a terminal null
+  // cursor is a runaway; completing on the final allowed page is normal.
+  if (!completed) {
+    throw new Error(`location provider exceeded ${maxPages} pages`);
+  }
+  return observations;
+}
+
+/**
+ * Sync every configured source for one local day. The master gate
+ * (`config.enabled === false`) short-circuits every path: no providers are
+ * contacted and nothing touches the filesystem. A provider failure fails
+ * that source only (state not advanced) and never blocks the others.
+ */
+export async function syncLocationDay(options: LocationDaySyncOptions): Promise<LocationSourceSyncResult[]> {
+  if (!options.config.enabled) return [];
+  const window = locationDayWindow(options.date, options.config.timezone);
+  const now = options.now ?? (() => new Date());
+  const resolveProvider = options.getProvider ?? getLocationProvider;
+
+  const results: LocationSourceSyncResult[] = [];
+  for (const source of options.config.sources) {
+    if (!source.enabled) {
+      results.push({
+        sourceId: source.id,
+        status: "skipped",
+        skipReason: "source-disabled",
+        fetched: 0,
+        dayWritten: false,
+        stateSaved: false,
+      });
+      continue;
+    }
+    const provider = resolveProvider(source.id);
+    if (provider === undefined) {
+      results.push({
+        sourceId: source.id,
+        status: "skipped",
+        skipReason: "provider-not-registered",
+        fetched: 0,
+        dayWritten: false,
+        stateSaved: false,
+      });
+      continue;
+    }
+
+    let observations: LocationObservation[];
+    try {
+      observations = await fetchWindow(provider, window, options);
+    } catch (error) {
+      results.push({
+        sourceId: source.id,
+        status: "failed",
+        fetched: 0,
+        dayWritten: false,
+        stateSaved: false,
+        ...(error instanceof Error && error.name === "AbortError"
+          ? { error: "aborted" }
+          : { error: sanitizeProviderError(error) }),
+      });
+      continue;
+    }
+
+    try {
+      options.signal?.throwIfAborted();
+      const segments = observationSegments(observations, window, {
+        retainCoordinates: options.config.retainCoordinates,
+      });
+      const payload = { observationCount: observations.length, segments, providerDisplayName: provider.displayName };
+      const merged = await mergeDaySources(options.memoryDir, source.id, options.date, payload);
+      let dayWritten = false;
+      if (Object.values(merged).some((entry) => entry.observationCount > 0)) {
+        const body = composeLocationDayBody(options.date, options.config.timezone, merged);
+        const meta = composeLocationDayMeta(options.date, options.config.timezone, merged, body);
+        dayWritten = await writeLocationDay(options.memoryDir, options.date, serializeLocationDay(meta, body));
+      }
+      // State advances only after the durable day write above succeeded.
+      await updateLocationSourceDay(
+        options.memoryDir,
+        source.id,
+        options.date,
+        payload,
+        now().toISOString(),
+      );
+      results.push({
+        sourceId: source.id,
+        status: "synced",
+        fetched: observations.length,
+        dayWritten,
+        stateSaved: true,
+      });
+    } catch (error) {
+      results.push({
+        sourceId: source.id,
+        status: "failed",
+        fetched: observations.length,
+        dayWritten: false,
+        stateSaved: false,
+        ...(error instanceof Error && error.name === "AbortError"
+          ? { error: "aborted" }
+          : { error: sanitizeProviderError(error) }),
+      });
+    }
+  }
+  return results;
+}
+
+/**
+ * The day-document source map: this source's fresh payload plus every OTHER
+ * source's last-known payload from sync state. Rendered documents therefore
+ * survive a re-sync of one source without re-fetching the others.
+ */
+async function mergeDaySources(
+  memoryDir: string,
+  sourceId: string,
+  date: string,
+  payload: { observationCount: number; segments: LocationSegment[] },
+): Promise<Record<string, LocationDaySourceEntry>> {
+  const state = await loadLocationSyncState(memoryDir);
+  const merged: Record<string, LocationDaySourceEntry> = {};
+  for (const [otherId, otherState] of Object.entries(state.sources)) {
+    if (otherId === sourceId) continue;
+    const day = otherState.days?.[date];
+    if (day === undefined) continue;
+    merged[otherId] = { ...day, segments: [...day.segments] };
+  }
+  merged[sourceId] = payload;
+  return merged;
+}

package/src/location/recall-exclusion.test.ts

@@ -0,0 +1,36 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { isSearchExcludedPath } from "../orchestration/generic-recall-paths.js";
+import { isLocationDayPath } from "../orchestration/orchestrator-helpers.js";
+
+const MEMORY_DIR = "/tmp/remnic-fixture-memory";
+
+test("isLocationDayPath matches only the top-level day-file shape", () => {
+  assert.equal(isLocationDayPath("locations/2026-08-17.md", MEMORY_DIR), true);
+  assert.equal(isLocationDayPath(`${MEMORY_DIR}/locations/2026-08-17.md`, MEMORY_DIR), true);
+  assert.equal(isLocationDayPath("facts/notes/locations.md", MEMORY_DIR), false);
+  assert.equal(isLocationDayPath("facts/locations/2026-08-17.md", MEMORY_DIR), false, "nested stays recallable");
+  assert.equal(isLocationDayPath("locations/2026-08-17.md"), true, "root-unaware best-effort fallback");
+  assert.equal(isLocationDayPath("locations/not-a-date.md"), false);
+});
+
+test("isSearchExcludedPath keeps location day documents out of generic search and recall", () => {
+  assert.equal(isSearchExcludedPath("locations/2026-08-17.md"), true);
+  assert.equal(
+    isSearchExcludedPath("qmd://collection/locations/2026-08-17.md", {}, "qmd"),
+    true,
+    "collection-qualified QMD paths are stripped and re-tested",
+  );
+  assert.equal(
+    isSearchExcludedPath(`${MEMORY_DIR}/namespaces/team/locations/2026-08-17.md`, { memoryDir: MEMORY_DIR }),
+    true,
+    "a namespaced day document is excluded via the namespace root",
+  );
+  assert.equal(isSearchExcludedPath("facts/locations-tips.md"), false);
+  assert.equal(
+    isSearchExcludedPath("namespaces/locations/facts/a.md", { memoryDir: MEMORY_DIR }),
+    false,
+    "a namespace literally named locations keeps its memories recallable",
+  );
+});

package/src/location/registry.test.ts

@@ -0,0 +1,74 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+  clearLocationProviders,
+  getLocationProvider,
+  listLocationProviders,
+  registerLocationProvider,
+} from "./registry.js";
+import type { LocationProvider } from "./types.js";
+
+function provider(overrides: Partial<LocationProvider> = {}): LocationProvider {
+  return {
+    id: "fixture-provider",
+    displayName: "Fixture Provider",
+    async verify() {
+      return { ok: true };
+    },
+    async fetchObservations() {
+      return { observations: [], nextCursor: null };
+    },
+    ...overrides,
+  };
+}
+
+test("registerLocationProvider registers, looks up, and lists sorted", () => {
+  clearLocationProviders();
+  registerLocationProvider(provider({ id: "zeta", displayName: "Z" }));
+  registerLocationProvider(provider({ id: "alpha", displayName: "A" }));
+  assert.deepEqual(listLocationProviders(), ["alpha", "zeta"]);
+  assert.equal(getLocationProvider("alpha")?.displayName, "A");
+  assert.equal(getLocationProvider("missing"), undefined);
+});
+
+test("registerLocationProvider rejects duplicate ids", () => {
+  clearLocationProviders();
+  registerLocationProvider(provider());
+  assert.throws(() => registerLocationProvider(provider()), /already registered/);
+});
+
+test("registerLocationProvider rejects invalid ids", () => {
+  clearLocationProviders();
+  assert.throws(() => registerLocationProvider(provider({ id: "" })), RangeError);
+  assert.throws(() => registerLocationProvider(provider({ id: "Upper" })), RangeError);
+  assert.throws(() => registerLocationProvider(provider({ id: "has space" })), RangeError);
+  assert.throws(() => registerLocationProvider(provider({ id: "../traversal" })), RangeError);
+  assert.throws(
+    () => registerLocationProvider(provider({ id: "x", displayName: "" as unknown as string })),
+    TypeError,
+  );
+  assert.throws(
+    () =>
+      registerLocationProvider(
+        provider({ id: "x", verify: undefined as unknown as LocationProvider["verify"] }),
+      ),
+    TypeError,
+  );
+  assert.throws(
+    () =>
+      registerLocationProvider(
+        provider({ id: "x", fetchObservations: undefined as unknown as LocationProvider["fetchObservations"] }),
+      ),
+    TypeError,
+  );
+  assert.deepEqual(listLocationProviders(), []);
+});
+
+test("clearLocationProviders resets the registry", () => {
+  clearLocationProviders();
+  registerLocationProvider(provider());
+  clearLocationProviders();
+  assert.deepEqual(listLocationProviders(), []);
+  assert.equal(getLocationProvider("fixture-provider"), undefined);
+});

package/src/location/registry.ts

@@ -0,0 +1,54 @@
+/**
+ * Location provider registry (issue #2044).
+ *
+ * In-memory only: host adapters register provider implementations at startup.
+ * Core never statically imports an optional provider package — the registry
+ * holds whatever the host wired in, and a configured source whose provider is
+ * not registered is skipped (not an error), so partial installs keep working.
+ */
+
+import type { LocationProvider } from "./types.js";
+
+const PROVIDER_ID_PATTERN = /^[a-z0-9][a-z0-9-]*$/;
+
+const providers = new Map<string, LocationProvider>();
+
+function assertValidProvider(provider: LocationProvider): void {
+  if (typeof provider !== "object" || provider === null) {
+    throw new TypeError("location provider must be an object");
+  }
+  if (typeof provider.id !== "string" || !PROVIDER_ID_PATTERN.test(provider.id)) {
+    throw new RangeError(
+      "location provider id must be a non-empty lowercase kebab string (a-z, 0-9, hyphens)",
+    );
+  }
+  if (typeof provider.displayName !== "string" || provider.displayName.trim().length === 0) {
+    throw new TypeError(`location provider '${provider.id}' must declare a non-empty displayName`);
+  }
+  if (typeof provider.verify !== "function" || typeof provider.fetchObservations !== "function") {
+    throw new TypeError(
+      `location provider '${provider.id}' must implement verify() and fetchObservations()`,
+    );
+  }
+}
+
+export function registerLocationProvider(provider: LocationProvider): void {
+  assertValidProvider(provider);
+  if (providers.has(provider.id)) {
+    throw new Error(`location provider '${provider.id}' is already registered`);
+  }
+  providers.set(provider.id, provider);
+}
+
+export function getLocationProvider(id: string): LocationProvider | undefined {
+  return typeof id === "string" ? providers.get(id) : undefined;
+}
+
+export function listLocationProviders(): string[] {
+  return [...providers.keys()].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0));
+}
+
+/** Test-only: reset the registry between cases. */
+export function clearLocationProviders(): void {
+  providers.clear();
+}

package/src/location/store.test.ts

@@ -0,0 +1,170 @@
+import assert from "node:assert/strict";
+import { mkdtemp, mkdir, rm, symlink, writeFile, readFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+
+import {
+  composeLocationDayBody,
+  composeLocationDayMeta,
+  emptyLocationSyncState,
+  listLocationDayDates,
+  loadLocationSyncState,
+  locationDayFilePath,
+  locationSyncStateFilePath,
+  parseLocationDaySummary,
+  serializeLocationDay,
+  updateLocationSourceDay,
+  writeLocationDay,
+} from "./store.js";
+import type { LocationDaySourceEntry } from "./store.js";
+
+function entry(overrides: Partial<LocationDaySourceEntry> = {}): LocationDaySourceEntry {
+  return {
+    observationCount: 1,
+    segments: [
+      {
+        startUtc: "2026-08-17T09:00:00.000Z",
+        endUtc: "2026-08-17T09:30:00.000Z",
+        place: { id: "home", label: "Home" },
+      },
+    ],
+    ...overrides,
+  };
+}
+
+test("day rendering is deterministic across source-map insertion order", () => {
+  const a: Record<string, LocationDaySourceEntry> = {
+    reitti: entry({ providerDisplayName: "Reitti" }),
+    other: entry({
+      providerDisplayName: "Other",
+      observationCount: 0,
+      segments: [],
+    }),
+  };
+  const b: Record<string, LocationDaySourceEntry> = {
+    other: entry({ providerDisplayName: "Other", observationCount: 0, segments: [] }),
+    reitti: entry({ providerDisplayName: "Reitti" }),
+  };
+  const render = (sources: Record<string, LocationDaySourceEntry>) => {
+    const body = composeLocationDayBody("2026-08-17", "UTC", sources);
+    return serializeLocationDay(composeLocationDayMeta("2026-08-17", "UTC", sources, body), body);
+  };
+  const first = render(a);
+  const second = render(b);
+  assert.equal(first, second, "identical content renders byte-identical regardless of key order");
+  const reparsed = parseLocationDaySummary(first);
+  assert.deepEqual(reparsed, {
+    date: "2026-08-17",
+    timezone: "UTC",
+    sources: ["other", "reitti"],
+    observationCount: 1,
+    contentHash: parseLocationDaySummary(second)!.contentHash,
+  });
+  assert.equal(parseLocationDaySummary("# not a location day\n"), null);
+});
+
+test("writeLocationDay is idempotent and read-back verifies content", async () => {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-location-store-"));
+  try {
+    const body = composeLocationDayBody("2026-08-17", "UTC", { reitti: entry() });
+    const serialized = serializeLocationDay(
+      composeLocationDayMeta("2026-08-17", "UTC", { reitti: entry() }, body),
+      body,
+    );
+    assert.equal(await writeLocationDay(memoryDir, "2026-08-17", serialized), true);
+    assert.equal(await writeLocationDay(memoryDir, "2026-08-17", serialized), false, "unchanged content rewrites nothing");
+    assert.equal(await readFile(locationDayFilePath(memoryDir, "2026-08-17"), "utf-8"), serialized);
+    assert.deepEqual(await listLocationDayDates(memoryDir), ["2026-08-17"]);
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+test("locationDayFilePath rejects traversal-shaped dates", () => {
+  assert.throws(() => locationDayFilePath("/tmp/x", "../../etc/passwd"), RangeError);
+  assert.throws(() => locationDayFilePath("/tmp/x", "2026-13-01"), RangeError);
+});
+
+test("store rejects a symlinked locations directory even when it resolves inside memoryDir", async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-location-symlink-"));
+  try {
+    const memoryDir = path.join(root, "memory");
+    const target = path.join(root, "elsewhere");
+    await mkdir(memoryDir, { recursive: true });
+    await mkdir(target, { recursive: true });
+    await symlink(target, path.join(memoryDir, "locations"));
+    await assert.rejects(
+      () => writeLocationDay(memoryDir, "2026-08-17", "---\nkind: location-day\n"),
+      /symlink/,
+    );
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
+test("store rejects a symlinked day file that escapes the memory dir", async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-location-symlinkfile-"));
+  try {
+    const memoryDir = path.join(root, "memory");
+    await mkdir(path.join(memoryDir, "locations"), { recursive: true });
+    await writeFile(path.join(root, "secret.md"), "outside", "utf-8");
+    await symlink(path.join(root, "secret.md"), path.join(memoryDir, "locations", "2026-08-17.md"));
+    await assert.rejects(
+      () => writeLocationDay(memoryDir, "2026-08-17", "---\nkind: location-day\n"),
+      /outside the memory dir|symbolic link/,
+    );
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
+test("sync state cold-starts on absence and corrupt JSON, then persists atomically", async () => {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-location-state-"));
+  try {
+    assert.deepEqual(await loadLocationSyncState(memoryDir), emptyLocationSyncState());
+    await mkdir(path.dirname(locationSyncStateFilePath(memoryDir)), { recursive: true });
+    await writeFile(locationSyncStateFilePath(memoryDir), "{not json", "utf-8");
+    assert.deepEqual(await loadLocationSyncState(memoryDir), emptyLocationSyncState());
+    await updateLocationSourceDay(memoryDir, "reitti", "2026-08-17", entry(), "2026-08-18T00:00:05.000Z");
+    const state = await loadLocationSyncState(memoryDir);
+    assert.equal(state.sources.reitti?.lastSyncedAtUtc, "2026-08-18T00:00:05.000Z");
+    assert.equal(state.sources.reitti?.days?.["2026-08-17"]?.observationCount, 1);
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+test("concurrent source state updates do not lose each other", async () => {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-location-concurrent-"));
+  try {
+    await Promise.all([
+      updateLocationSourceDay(memoryDir, "reitti", "2026-08-17", entry(), "2026-08-18T00:00:01.000Z"),
+      updateLocationSourceDay(memoryDir, "other", "2026-08-17", entry({ observationCount: 2 }), "2026-08-18T00:00:02.000Z"),
+      updateLocationSourceDay(memoryDir, "third", "2026-08-16", entry(), "2026-08-18T00:00:03.000Z"),
+    ]);
+    const state = await loadLocationSyncState(memoryDir);
+    assert.deepEqual(Object.keys(state.sources).sort(), ["other", "reitti", "third"]);
+    assert.equal(state.sources.reitti?.days?.["2026-08-17"]?.observationCount, 1);
+    assert.equal(state.sources.other?.days?.["2026-08-17"]?.observationCount, 2);
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});
+
+test("day payloads prune beyond the retention cap, newest kept", async () => {
+  const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-location-prune-"));
+  try {
+    for (let day = 0; day < 95; day += 1) {
+      const date = new Date(Date.UTC(2026, 5, 1) + day * 86_400_000).toISOString().slice(0, 10);
+      await updateLocationSourceDay(memoryDir, "reitti", date, entry(), "2026-09-01T00:00:00.000Z");
+    }
+    const state = await loadLocationSyncState(memoryDir);
+    const days = Object.keys(state.sources.reitti?.days ?? {});
+    assert.equal(days.length, 90);
+    assert.equal(days.includes("2026-06-01"), false, "oldest days pruned");
+    assert.equal(days.includes("2026-06-06"), true, "newest 90 days kept");
+  } finally {
+    await rm(memoryDir, { recursive: true, force: true });
+  }
+});