@tpsdev-ai/flair 0.52.0 → 0.54.1

package/dist/resources/OrgEventCatchup.js

@@ -1,24 +1,94 @@
 /**
- * OrgEventCatchup.ts — Custom resource for filtered event retrieval.
+ * OrgEventCatchup.ts — Paginated catch-up against a per-agent watermark.
  *
- * GET /OrgEventCatchup/{participantId}?since=<ISO timestamp>
+ * GET  /OrgEventCatchup/{participantId}?after=<position>&since=<ISO>&limit=50
+ * POST /OrgEventCatchup/{participantId}  { position }
  *
  * Returns events where:
  *   - targetIds includes participantId OR targetIds is empty/null
- *   - createdAt >= since
- * Sorted by createdAt ascending (oldest first for in-order processing).
- * Limit 50 events max.
+ *   - position > after (after defaults to the durable per-agent watermark)
+ *   - optional `since` is an extra createdAt floor, never the primary cursor
+ *
+ * Ordered by position ascending. Page-sized, never silently truncated
+ * (`hasMore` + `nextAfter` — the caller pages until drained). Watermark
+ * advances on ack (POST), not on read — at-least-once; consumers must be
+ * idempotent (flair#931).
  */
 import { Resource, databases } from "harper";
 import { allowVerified, resolveAgentAuth } from "./agent-auth.js";
+import { initialPosition, ORG_EVENT_STREAM, parsePageSize, positionAfterTimestamp, } from "./agent-read-position-lib.js";
+import { advanceReadPosition, defaultReadPositionTable, ensureReadPosition, getReadPosition, } from "./agent-read-position.js";
+import { catchupSeekTimestamp, collectCatchupEvents, pageCatchupEvents, } from "./org-event-catchup-lib.js";
+function queryValue(pathInfo, name) {
+    if (typeof pathInfo !== "object" || pathInfo === null)
+        return null;
+    return pathInfo.conditions?.find((c) => c.attribute === name)?.value ?? null;
+}
+function pathParticipantId(pathInfo, resource) {
+    return ((typeof pathInfo === "object" && pathInfo !== null ? pathInfo.id : null) ??
+        (typeof pathInfo === "string" ? pathInfo : null) ??
+        resource.getId?.() ??
+        null);
+}
+function ownerDenied(auth, participantId) {
+    if (auth.kind === "anonymous")
+        return true;
+    if (auth.kind === "agent" && !auth.isAdmin && auth.agentId !== participantId)
+        return true;
+    return false;
+}
+function forbidden() {
+    return new Response(JSON.stringify({ error: "forbidden: can only fetch events for yourself" }), { status: 403, headers: { "Content-Type": "application/json" } });
+}
+async function resolveAfter(participantId, pathInfo, ctx) {
+    const afterParam = queryValue(pathInfo, "after");
+    const since = queryValue(pathInfo, "since");
+    if (since) {
+        const sinceDate = new Date(since);
+        if (isNaN(sinceDate.getTime())) {
+            throw new Response(JSON.stringify({ error: "invalid since timestamp" }), { status: 400, headers: { "Content-Type": "application/json" } });
+        }
+    }
+    const table = defaultReadPositionTable();
+    const watermark = await getReadPosition(table, participantId, ORG_EVENT_STREAM, ctx);
+    if (afterParam != null && afterParam !== "") {
+        return { after: afterParam, watermark, since };
+    }
+    if (watermark) {
+        return { after: watermark, watermark, since };
+    }
+    if (since) {
+        // Legacy explicit since, no watermark yet — do not invent a "now" floor
+        // that would hide the window the caller asked for.
+        return { after: positionAfterTimestamp(since), watermark: null, since };
+    }
+    const created = await ensureReadPosition(table, participantId, ORG_EVENT_STREAM, initialPosition(), ctx);
+    return { after: created, watermark: created, since: null };
+}
+async function searchOrgEvents(seekTs) {
+    const table = databases.flair?.OrgEvent;
+    if (!table?.search) {
+        async function* empty() { }
+        return empty();
+    }
+    const query = seekTs
+        ? {
+            conditions: [{ attribute: "createdAt", comparator: "greater_than_equal", value: seekTs }],
+        }
+        : undefined;
+    return table.search(query);
+}
 export class OrgEventCatchup extends Resource {
     // Self-authorize via the Ed25519 agent verify (auth reshape removes the gate's
     // admin elevation). Any verified agent may catch up; participant scoping is in
-    // get(). Uses getContext().request — the reliable v5 path (this.request is not
-    // populated on Harper v5 Resources).
+    // get()/post(). Uses getContext().request — the reliable v5 path (this.request
+    // is not populated on Harper v5 Resources).
     async allowRead() {
         return allowVerified(this.getContext?.());
     }
+    async allowCreate() {
+        return allowVerified(this.getContext?.());
+    }
     // HarperDB calls get(pathInfo, context) where pathInfo is the URL segment after /OrgEventCatchup/
     async get(pathInfo) {
         // Harper v5 does not populate this.request on Resource subclasses —
@@ -27,55 +97,64 @@ export class OrgEventCatchup extends Resource {
         // any).request` read was always undefined, so the ownership check below
         // never ran — fail-open cross-agent read).
         const auth = await resolveAgentAuth(this.getContext?.());
-        // Harper routes /OrgEventCatchup/{id} with pathInfo.id as the path segment
-        const participantId = (typeof pathInfo === "object" && pathInfo !== null ? pathInfo.id : null) ??
-            (typeof pathInfo === "string" ? pathInfo : null) ??
-            this.getId?.() ??
-            null;
+        const ctx = this.getContext?.();
+        const participantId = pathParticipantId(pathInfo, this);
         if (!participantId) {
             return new Response(JSON.stringify({ error: "participantId required in path: GET /OrgEventCatchup/{participantId}" }), { status: 400, headers: { "Content-Type": "application/json" } });
         }
         // Auth: internal calls and admins pass unfiltered; a verified agent may only
         // fetch its own catchup feed; anonymous is denied. allowRead() already
         // blocks anonymous HTTP, but this handler must fail closed on its own too.
-        if (auth.kind === "anonymous") {
-            return new Response(JSON.stringify({ error: "forbidden: can only fetch events for yourself" }), { status: 403, headers: { "Content-Type": "application/json" } });
-        }
-        if (auth.kind === "agent" && !auth.isAdmin && auth.agentId !== participantId) {
-            return new Response(JSON.stringify({ error: "forbidden: can only fetch events for yourself" }), { status: 403, headers: { "Content-Type": "application/json" } });
+        if (ownerDenied(auth, participantId))
+            return forbidden();
+        let resolved;
+        try {
+            resolved = await resolveAfter(participantId, pathInfo, ctx);
         }
-        // Harper parses query params into pathInfo.conditions array:
-        // e.g. ?since=value → conditions: [{attribute:"since", value:"...", comparator:"equals"}]
-        // pathInfo.id is the URL path segment (participantId).
-        const since = (typeof pathInfo === "object" && pathInfo !== null
-            ? pathInfo.conditions?.find((c) => c.attribute === "since")?.value ?? null
-            : null);
-        if (!since) {
-            return new Response(JSON.stringify({ error: "since query parameter required (ISO timestamp)" }), { status: 400, headers: { "Content-Type": "application/json" } });
+        catch (err) {
+            if (err instanceof Response)
+                return err;
+            throw err;
         }
-        const sinceDate = new Date(since);
-        if (isNaN(sinceDate.getTime())) {
-            return new Response(JSON.stringify({ error: "invalid since timestamp" }), { status: 400, headers: { "Content-Type": "application/json" } });
+        const pageSize = parsePageSize(queryValue(pathInfo, "limit"));
+        const seekTs = catchupSeekTimestamp(resolved.after, resolved.since);
+        const ordered = await collectCatchupEvents(await searchOrgEvents(seekTs), {
+            participantId,
+            after: resolved.after,
+            since: resolved.since,
+        });
+        const paged = pageCatchupEvents(ordered, resolved.after, pageSize);
+        return {
+            events: paged.page,
+            watermark: resolved.watermark,
+            after: resolved.after,
+            nextAfter: paged.nextAfter,
+            hasMore: paged.hasMore,
+            pageSize,
+        };
+    }
+    /**
+     * Advance-on-ack. Body: `{ position }` (the last delivered event's
+     * `position`, or `nextAfter` from a drained page). Monotonic.
+     */
+    async post(content, pathInfo) {
+        const auth = await resolveAgentAuth(this.getContext?.());
+        const ctx = this.getContext?.();
+        const participantId = pathParticipantId(pathInfo, this) ??
+            (content && typeof content === "object" ? content.participantId ?? content.agentId : null) ??
+            null;
+        if (!participantId) {
+            return new Response(JSON.stringify({ error: "participantId required in path: POST /OrgEventCatchup/{participantId}" }), { status: 400, headers: { "Content-Type": "application/json" } });
         }
-        // Query all OrgEvents and filter in-memory
-        const results = [];
-        for await (const event of databases.flair.OrgEvent.search()) {
-            // Filter by createdAt >= since
-            if (!event.createdAt || event.createdAt < since)
-                continue;
-            // Filter by targetIds: includes participantId OR empty/null
-            const targets = event.targetIds;
-            const isTargeted = !targets || targets.length === 0 || targets.includes(participantId);
-            if (!isTargeted)
-                continue;
-            // Skip expired events
-            if (event.expiresAt && new Date(event.expiresAt) < new Date())
-                continue;
-            results.push(event);
+        if (ownerDenied(auth, participantId))
+            return forbidden();
+        const position = content && typeof content === "object"
+            ? String(content.position ?? content.ackThrough ?? "")
+            : "";
+        if (!position) {
+            return new Response(JSON.stringify({ error: "position required to ack catch-up (advance-on-ack)" }), { status: 400, headers: { "Content-Type": "application/json" } });
         }
-        // Sort ascending by createdAt (oldest first)
-        results.sort((a, b) => (a.createdAt || "").localeCompare(b.createdAt || ""));
-        // Limit to 50
-        return results.slice(0, 50);
+        const result = await advanceReadPosition(defaultReadPositionTable(), participantId, ORG_EVENT_STREAM, position, ctx);
+        return { participantId, stream: ORG_EVENT_STREAM, ...result };
     }
 }

package/dist/resources/agent-read-position-lib.js

@@ -0,0 +1,83 @@
+/**
+ * agent-read-position-lib.ts — pure per-agent read-position (watermark) helpers.
+ *
+ * This is the reusable catch-up floor (flair#931), not OrgEvent-table plumbing.
+ * OrgEventCatchup is the first consumer; light-comms (#1583) inherits the same
+ * (agentId, stream) → position primitive when the board lands.
+ *
+ * Position is a monotonic total order, NOT a wall-clock `since`:
+ *   `${createdAt}\t${id}`
+ * ISO-8601 timestamps sort lexicographically; `id` breaks same-ms ties so a
+ * sort without a tie-break cannot drop an event. Compare with `<` / `>`.
+ *
+ * Harper-free so unit tests can pin encoding, paging, and init without a store.
+ */
+import { compareKey } from "./sort-comparators.js";
+/** First shipped stream. Additional streams (message-board, …) are additive. */
+export const ORG_EVENT_STREAM = "org-event";
+export const POSITION_SEP = "\t";
+export const DEFAULT_CATCHUP_PAGE_SIZE = 50;
+export const MAX_CATCHUP_PAGE_SIZE = 500;
+/** Env: bounded backfill on first watermark. Unset = 24h (retired window, once). `0` = "now". */
+export const CATCHUP_BACKFILL_MS_ENV = "FLAIR_CATCHUP_BACKFILL_MS";
+export const DEFAULT_CATCHUP_BACKFILL_MS = 24 * 3600_000;
+export function readPositionId(agentId, stream) {
+    return `${agentId}:${stream}`;
+}
+/** Total-order position for a record that has `createdAt` + `id`. */
+export function recordPosition(record) {
+    return `${record.createdAt ?? ""}${POSITION_SEP}${record.id ?? ""}`;
+}
+/** Exclusive cursor just after every record at-or-before this ISO timestamp. */
+export function positionAfterTimestamp(iso) {
+    return `${iso}${POSITION_SEP}`;
+}
+export function comparePosition(a, b) {
+    return compareKey(a, b);
+}
+/** Leading createdAt of a position, or null if the cursor has no timestamp. */
+export function createdAtFloorFromPosition(position) {
+    if (!position)
+        return null;
+    const sep = position.indexOf(POSITION_SEP);
+    const ts = sep === -1 ? position : position.slice(0, sep);
+    return ts || null;
+}
+export function laterTimestamp(a, b) {
+    if (!a)
+        return b || null;
+    if (!b)
+        return a;
+    return a >= b ? a : b;
+}
+export function parseBackfillMs(raw = process.env[CATCHUP_BACKFILL_MS_ENV]) {
+    if (raw === undefined || raw === "")
+        return DEFAULT_CATCHUP_BACKFILL_MS;
+    const n = Number(raw);
+    if (!Number.isFinite(n) || n < 0)
+        return DEFAULT_CATCHUP_BACKFILL_MS;
+    return n;
+}
+/**
+ * Sane first watermark. Default a 24h bounded backfill (the retired bootstrap
+ * window, applied once); `FLAIR_CATCHUP_BACKFILL_MS=0` is Flint's "now"
+ * (fresh agent does not replay history). After init the durable position governs.
+ */
+export function initialPosition(nowMs = Date.now(), backfillMs = parseBackfillMs()) {
+    const floor = Math.max(0, nowMs - Math.max(0, backfillMs));
+    return positionAfterTimestamp(new Date(floor).toISOString());
+}
+export function parsePageSize(raw, fallback = DEFAULT_CATCHUP_PAGE_SIZE, max = MAX_CATCHUP_PAGE_SIZE) {
+    const n = typeof raw === "number" ? raw : raw != null && raw !== "" ? Number(raw) : fallback;
+    if (!Number.isFinite(n))
+        return fallback;
+    return Math.min(max, Math.max(1, Math.floor(n)));
+}
+/** Exclusive `after`, already-sorted by position ascending. Never silent-truncates. */
+export function pageAfter(ordered, after, pageSize) {
+    const slice = ordered.filter((row) => comparePosition(row.position, after) > 0);
+    const hasMore = slice.length > pageSize;
+    const page = hasMore ? slice.slice(0, pageSize) : slice;
+    const nextAfter = page.length > 0 ? page[page.length - 1].position : after;
+    return { page, hasMore, nextAfter };
+}

package/dist/resources/agent-read-position.js

@@ -0,0 +1,120 @@
+/**
+ * agent-read-position.ts — durable per-agent read-position (watermark) store.
+ *
+ * Storage + monotonic advance for resources/agent-read-position-lib.ts.
+ * Owner-scoping lives in the HTTP resource / catch-up handler; this module
+ * is the internal contract for raw table access (Harper table I/O bypasses
+ * Resource rules).
+ *
+ * Advance is at-least-once (advance-on-ack): only move forward, never back.
+ * Same-process concurrent advances serialize on a per-id promise tail so two
+ * acks cannot clobber a higher watermark with a lower one.
+ */
+import { databases } from "harper";
+import { withDetachedTxn } from "./table-helpers.js";
+import { comparePosition, readPositionId } from "./agent-read-position-lib.js";
+export function defaultReadPositionTable() {
+    try {
+        const table = databases.flair?.AgentReadPosition;
+        if (!table || typeof table.get !== "function" || typeof table.put !== "function")
+            return null;
+        return table;
+    }
+    catch {
+        return null;
+    }
+}
+const tails = new Map();
+function enqueue(id, work) {
+    const prev = tails.get(id) ?? Promise.resolve();
+    const next = prev.then(work, work);
+    tails.set(id, next);
+    return next.finally(() => {
+        if (tails.get(id) === next)
+            tails.delete(id);
+    });
+}
+async function readRow(table, id, ctx) {
+    try {
+        const row = await withDetachedTxn(ctx, () => table.get(id));
+        return row && typeof row.position === "string" ? row : null;
+    }
+    catch {
+        return null;
+    }
+}
+export async function getReadPosition(table, agentId, stream, ctx) {
+    if (!table)
+        return null;
+    const row = await readRow(table, readPositionId(agentId, stream), ctx);
+    return row?.position ?? null;
+}
+/**
+ * Return the stored watermark, creating it at `initial` when absent.
+ * Concurrent first-writers serialize; the stored row wins if one appears.
+ */
+export async function ensureReadPosition(table, agentId, stream, initial, ctx, now = () => new Date().toISOString()) {
+    if (!table)
+        return initial;
+    const id = readPositionId(agentId, stream);
+    return enqueue(id, async () => {
+        const existing = await readRow(table, id, ctx);
+        if (existing)
+            return existing.position;
+        const row = {
+            id,
+            agentId,
+            stream,
+            position: initial,
+            updatedAt: now(),
+        };
+        try {
+            await withDetachedTxn(ctx, () => table.put(row));
+        }
+        catch {
+            const raced = await readRow(table, id, ctx);
+            if (raced)
+                return raced.position;
+            return initial;
+        }
+        return initial;
+    });
+}
+/**
+ * Monotonic advance. A lower-or-equal ack is a no-op. Serialized per
+ * (agent, stream) so concurrent catch-up acks cannot regress the watermark.
+ */
+export async function advanceReadPosition(table, agentId, stream, position, ctx, now = () => new Date().toISOString()) {
+    if (!position) {
+        const current = table ? await getReadPosition(table, agentId, stream, ctx) : null;
+        return { position: current ?? "", advanced: false };
+    }
+    if (!table)
+        return { position, advanced: false };
+    const id = readPositionId(agentId, stream);
+    return enqueue(id, async () => {
+        const existing = await readRow(table, id, ctx);
+        const current = existing?.position ?? "";
+        if (current && comparePosition(position, current) <= 0) {
+            return { position: current, advanced: false };
+        }
+        const row = {
+            id,
+            agentId,
+            stream,
+            position,
+            updatedAt: now(),
+        };
+        try {
+            await withDetachedTxn(ctx, () => table.put(row));
+        }
+        catch {
+            const raced = await readRow(table, id, ctx);
+            if (raced && comparePosition(raced.position, position) >= 0) {
+                return { position: raced.position, advanced: false };
+            }
+            return { position: current || position, advanced: false };
+        }
+        return { position, advanced: true };
+    });
+}

package/dist/resources/embeddings-boot.js

@@ -175,6 +175,23 @@ export function resolveEmbedThreads(env = process.env, cores = availableParallel
     const safeCores = Number.isFinite(cores) && cores >= 1 ? Math.floor(cores) : 1;
     return Math.max(1, safeCores - 1);
 }
+/**
+ * llama.cpp GPU-layer offload passed to HFE `register({config:{gpuLayers}})`.
+ *
+ * DEFAULT IS UNCHANGED. Unset / empty / non-integer / negative → `undefined`,
+ * and the register() call OMITS the field so HFE keeps its own default of 0
+ * (CPU only). This is a measurement pin for the ingest-throughput bench
+ * (flair#1436 / #1437), not a product default change. #1437 is the decision
+ * about whether to detect-and-default gpuLayers; this function must not
+ * pre-empt it.
+ *
+ * Override: `FLAIR_EMBED_GPU_LAYERS` — a non-negative integer, env-only
+ * (same persist-path reason as `FLAIR_EMBED_THREADS`). 0 = CPU only; 99 =
+ * full offload (the Metal cell). Invalid values fall through to omit.
+ */
+export function resolveEmbedGpuLayers(env = process.env) {
+    return parseNonNegativeInt(env.FLAIR_EMBED_GPU_LAYERS);
+}
 function parsePositiveInt(raw) {
     if (raw == null)
         return undefined;
@@ -186,6 +203,17 @@ function parsePositiveInt(raw) {
         return undefined;
     return n;
 }
+function parseNonNegativeInt(raw) {
+    if (raw == null)
+        return undefined;
+    const trimmed = raw.trim();
+    if (trimmed === "")
+        return undefined;
+    const n = Number(trimmed);
+    if (!Number.isInteger(n) || n < 0)
+        return undefined;
+    return n;
+}
 let registered = false;
 /**
  * Register the embedding backend. Idempotent within a process (mirrors
@@ -200,6 +228,9 @@ export async function registerEmbeddingsBackend() {
         const { register } = await import("harper-fabric-embeddings");
         const modelPath = benchModelPathOverride();
         const threads = resolveEmbedThreads();
+        // Omit when unset so HFE's default of 0 is unchanged (flair#1436 measurement
+        // pin; #1437 is the default-change decision). Never pass a synthesized default.
+        const gpuLayers = resolveEmbedGpuLayers();
         await register({
             logicalName: LOGICAL_NAME,
             kind: "embedding",
@@ -208,6 +239,7 @@ export async function registerEmbeddingsBackend() {
                     ? { modelPath, pooling: EMBEDDING_POOLING }
                     : { modelName: MODEL_NAME, modelsDir: resolveModelsDir(), pooling: EMBEDDING_POOLING }),
                 threads,
+                ...(gpuLayers !== undefined ? { gpuLayers } : {}),
             },
         });
     }

package/dist/resources/federation-peer-liveness.js

@@ -0,0 +1,73 @@
+/**
+ * Pure classifier for federation peer liveness — no Harper imports.
+ *
+ * flair#1499: HealthDetail used to count `status === "connected"` (a value
+ * pairing never writes — spokes stay `paired` even after a successful
+ * lastSyncAt advance) and then take the oldest lastSyncAt across ALL peers,
+ * including revoked. A healthy hub synced a minute ago plus a revoked row
+ * from June therefore rendered as "0 connected" and fired
+ * "federation peers all disconnected >24h".
+ *
+ * Three contact states, plus revoked (which never drives the staleness
+ * warning). Same class as #988: missing/unreadable evidence is UNKNOWN,
+ * never DISCONNECTED.
+ *
+ *   (a) real lastSyncAt within the window → connected
+ *   (b) real lastSyncAt older than the window → disconnected
+ *   (c) no parseable lastSyncAt → unknown (never rendered as (b))
+ *   revoked stored status → revoked (excluded from the >24h warning)
+ */
+export const PEER_STALE_MS = 24 * 3600 * 1000;
+export const FEDERATION_PEERS_ALL_DISCONNECTED_WARNING = "federation peers all disconnected >24h";
+/** Parse a written last-contact stamp. Missing / unparseable → null, never 0/epoch. */
+export function parseLastContactMs(lastSyncAt) {
+    if (typeof lastSyncAt !== "string")
+        return null;
+    const trimmed = lastSyncAt.trim();
+    if (!trimmed)
+        return null;
+    const t = Date.parse(trimmed);
+    return Number.isFinite(t) ? t : null;
+}
+export function classifyPeerLiveness(peer, nowMs, windowMs = PEER_STALE_MS) {
+    if (peer.status === "revoked")
+        return "revoked";
+    const ts = parseLastContactMs(peer.lastSyncAt);
+    if (ts === null)
+        return "unknown";
+    // Future / equal-now stamps are contact within the window (clock skew).
+    if (nowMs - ts > windowMs)
+        return "disconnected";
+    return "connected";
+}
+export function summarizePeerLiveness(peers, nowMs, windowMs = PEER_STALE_MS) {
+    let connected = 0;
+    let disconnected = 0;
+    let revoked = 0;
+    let unknown = 0;
+    for (const peer of peers) {
+        const liveness = classifyPeerLiveness(peer, nowMs, windowMs);
+        if (liveness === "connected")
+            connected++;
+        else if (liveness === "disconnected")
+            disconnected++;
+        else if (liveness === "revoked")
+            revoked++;
+        else
+            unknown++;
+    }
+    const nonRevoked = peers.length - revoked;
+    return {
+        total: peers.length,
+        connected,
+        disconnected,
+        revoked,
+        unknown,
+        allNonRevokedDisconnected: nonRevoked > 0 && disconnected === nonRevoked,
+    };
+}
+export function federationPeersAllDisconnectedWarning(summary) {
+    if (!summary.allNonRevokedDisconnected)
+        return null;
+    return { level: "warn", message: FEDERATION_PEERS_ALL_DISCONNECTED_WARNING };
+}