@tpsdev-ai/flair 0.17.0 → 0.19.0

package/dist/resources/Relationship.js

@@ -1,6 +1,7 @@
 import { databases } from "@harperfast/harper";
-import { resolveAgentAuth } from "./agent-auth.js";
+import { resolveAgentAuth, allowVerified } from "./agent-auth.js";
 import { checkRateLimit, rateLimitResponse } from "./rate-limiter.js";
+const NOT_FOUND = () => new Response(JSON.stringify({ error: "not found" }), { status: 404, headers: { "Content-Type": "application/json" } });
 /**
  * Relationship resource — entity-to-entity relationships with temporal validity.
  *
@@ -13,6 +14,53 @@ import { checkRateLimit, rateLimitResponse } from "./rate-limiter.js";
  * Admin agents can query across all agents.
  */
 export class Relationship extends databases.flair.Relationship {
+    /**
+     * Self-authorize now that the global gate is non-rejecting (memory-soul-
+     * read-gate family fix, ops-oox7 — same pattern as Memory.ts/Soul.ts/
+     * WorkspaceState.ts). Closes the same P0 leak: Harper routes
+     * `GET /Relationship/<id>` to get() and the collection describe
+     * (`GET /Relationship`) outside search(), so neither was gated before this
+     * fix — an anonymous caller got a 200 with full record content. Per-record
+     * ownership scoping happens in get() below; the collection scope is still
+     * in search().
+     */
+    allowRead() { return allowVerified(this.getContext?.()); }
+    /**
+     * Override get() to scope by-id reads the same way search() scopes
+     * collection reads (memory-soul-read-gate family fix). Never distinguishes
+     * "doesn't exist" from "exists but not yours" — both return 404, never
+     * 403, so a denied caller can't use get() to enumerate other agents'
+     * relationship ids.
+     */
+    async get(target) {
+        // Collection / query reads arrive as a RequestTarget with
+        // `isCollection === true`, and are governed by search() (same owner
+        // scoping). Only a genuine by-id get is ownership-checked below — see
+        // Memory.ts's get() for the full rationale (same bug class: without this
+        // guard, a query's RequestTarget would flow into super.get(), return the
+        // whole result set, and the single-record ownership check below would
+        // find no `.agentId` on it).
+        if (!target || (typeof target === "object" && target.isCollection)) {
+            return this.search(target);
+        }
+        const auth = await resolveAgentAuth(this.getContext?.());
+        // Anonymous by-id read is already blocked at the allowRead() gate (403);
+        // this is defense-in-depth if get() is ever reached directly.
+        if (auth.kind === "anonymous") {
+            return NOT_FOUND();
+        }
+        // Trusted internal call or admin agent — unfiltered, unchanged behavior.
+        if (auth.kind === "internal" || (auth.kind === "agent" && auth.isAdmin)) {
+            return super.get(target);
+        }
+        // Non-admin agent: only its own relationships.
+        const record = await super.get(target);
+        if (!record)
+            return NOT_FOUND();
+        if (record.agentId !== auth.agentId)
+            return NOT_FOUND();
+        return record;
+    }
     async search(query) {
         const auth = await resolveAgentAuth(this.getContext?.());
         // Anonymous HTTP must NOT read relationships (previously `!authAgent` was

package/dist/resources/SemanticSearch.js

@@ -4,6 +4,7 @@ import { getEmbedding, getMode } from "./embeddings-provider.js";
 import { patchRecord, withDetachedTxn } from "./table-helpers.js";
 import { checkRateLimit, rateLimitResponse } from "./rate-limiter.js";
 import { wrapUntrusted } from "./content-safety.js";
+import { isRerankEnabled, getRerankTopN, getRerankBudgetMs, getRerankMinCandidates, rerankCandidates, } from "./rerank-provider.js";
 // Temporal decay + relevance scoring (incl. the OPS-AYGD retrievalBoost cap +
 // relevance floor) lives in ./scoring.ts — a Harper-free module so it can be
 // unit-tested directly (see test/unit/temporal-scoring.test.ts).
@@ -170,6 +171,16 @@ export class SemanticSearch extends Resource {
         }
         const results = [];
         const hybrid = hybridEnabled();
+        // When the reranker is on, widen the legacy HNSW fetch so it has a deeper
+        // pool to re-score (retrieve topN → rerank → slice to limit). Decoupled
+        // from CANDIDATE_MULTIPLIER so composite re-ranking keeps its existing
+        // headroom. Scoped to the legacy (non-hybrid) vector path below — the
+        // hybrid path's candidate pool is already governed by CANDIDATE_MULTIPLIER
+        // (semantic leg) + SEM_LIMIT (BM25 leg) via RRF union (ops-i39b); the
+        // reranker still applies to its output further down regardless of which
+        // path produced `filteredResults`.
+        const rerankOn = isRerankEnabled();
+        const rerankTopN = getRerankTopN();
         if (hybrid) {
             // ─── BM25 + union-RRF hybrid path (ops-i39b) ─────────────────────────
             // 1. Semantic candidates via HNSW (unchanged fetch). 2. BM25 lexical pass
@@ -284,8 +295,10 @@ export class SemanticSearch extends Resource {
             }
         }
         else if (qEmb) {
-            // ─── HNSW vector search path ───────────────────────────────────────────
-            const candidateLimit = limit * CANDIDATE_MULTIPLIER;
+            // ─── HNSW vector search path (legacy, hybrid flag OFF) ────────────────
+            const candidateLimit = rerankOn
+                ? Math.max(limit * CANDIDATE_MULTIPLIER, rerankTopN)
+                : limit * CANDIDATE_MULTIPLIER;
             const query = {
                 sort: { attribute: "embedding", target: qEmb, distance: "cosine" },
                 select: ["id", "agentId", "content", "contentHash", "visibility", "tags", "durability",
@@ -387,7 +400,32 @@ export class SemanticSearch extends Resource {
         if (minScore > 0) {
             filteredResults = filteredResults.filter((r) => r._score >= minScore);
         }
-        filteredResults.sort((a, b) => b._score - a._score);
+        // ─── Cross-encoder rerank (best-effort, fail-open to vector order) ───────
+        // Re-scores query+candidate TOGETHER (cross-attention the pooled embedding
+        // can't do) and reorders before the final slice. Reorders whatever
+        // `filteredResults` the retrieval stage above produced — legacy HNSW-only
+        // OR the BM25+union-RRF hybrid path (ops-i39b) — since both converge into
+        // the same `results`/`filteredResults` shape before this point; hybrid
+        // retrieves+fuses, the reranker only reorders the fused set. Still gated
+        // on `qEmb` (an embedding was actually generated); the pure keyword-only
+        // fallback (no qEmb at all) is untouched either way. The reranker
+        // overwrites `_score` with the rerank score (so margin measurement reads it)
+        // and preserves the semantic score as `_semScore`; `_rawScore` is left as-is
+        // so recall-bench's scoring:"raw" path stays reproducible. On init failure,
+        // timeout, or any throw, rerankCandidates returns the input UNCHANGED and we
+        // fall through to the existing vector-order sort — recall is never blocked.
+        if (rerankOn && qEmb && q && filteredResults.length >= getRerankMinCandidates()) {
+            // Pre-sort by vector order so the topN fed to the reranker is the most
+            // semantically-promising slice (filteredResults isn't sorted yet here).
+            filteredResults.sort((a, b) => b._score - a._score);
+            filteredResults = await rerankCandidates(String(q), filteredResults, {
+                topN: rerankTopN,
+                budgetMs: getRerankBudgetMs(),
+            });
+        }
+        else {
+            filteredResults.sort((a, b) => b._score - a._score);
+        }
         const topResults = filteredResults.slice(0, limit);
         // Async hit tracking — don't block the response
         const now = new Date().toISOString();

package/dist/resources/Soul.js

@@ -1,5 +1,5 @@
 import { databases } from "@harperfast/harper";
-import { resolveAgentAuth } from "./agent-auth.js";
+import { resolveAgentAuth, allowVerified } from "./agent-auth.js";
 const FORBIDDEN = (msg) => new Response(JSON.stringify({ error: msg }), { status: 403, headers: { "Content-Type": "application/json" } });
 const UNAUTH = () => new Response(JSON.stringify({ error: "authentication required" }), { status: 401, headers: { "Content-Type": "application/json" } });
 /**
@@ -19,6 +19,17 @@ async function enforceWriteAuth(self, data) {
     return null;
 }
 export class Soul extends databases.flair.Soul {
+    /**
+     * Self-authorize now that the global gate is non-rejecting. Closes the P0
+     * leak: Harper routes `GET /Soul/<id>` to get() and the collection
+     * describe (`GET /Soul`) to a path outside search()/allow* — neither was
+     * gated before this fix, so an anonymous caller got a 200 with full soul
+     * content. Deliberately NO get() override / per-agent scoping on top of
+     * this: souls are identity/discovery data, intentionally readable by any
+     * verified agent — same posture as Agent.ts's allowRead. Write ownership
+     * is unaffected — enforceWriteAuth() below already gates post()/put().
+     */
+    allowRead() { return allowVerified(this.getContext?.()); }
     async post(content, context) {
         const denied = await enforceWriteAuth(this, content);
         if (denied)

package/dist/resources/WorkspaceState.js

@@ -8,15 +8,64 @@
  * Use this.getContext() to access request context (tpsAgent, tpsAgentIsAdmin).
  */
 import { databases } from "@harperfast/harper";
-import { resolveAgentAuth } from "./agent-auth.js";
+import { resolveAgentAuth, allowVerified } from "./agent-auth.js";
 const FORBIDDEN = (msg) => new Response(JSON.stringify({ error: msg }), { status: 403, headers: { "Content-Type": "application/json" } });
 const UNAUTH = () => new Response(JSON.stringify({ error: "authentication required" }), { status: 401, headers: { "Content-Type": "application/json" } });
+const NOT_FOUND = () => new Response(JSON.stringify({ error: "not found" }), { status: 404, headers: { "Content-Type": "application/json" } });
 export class WorkspaceState extends databases.flair.WorkspaceState {
     /** Auth verdict from the request context. internal = trusted in-process call;
      *  agent = verified Ed25519; anonymous = HTTP with no valid agent → deny. */
     _auth() {
         return resolveAgentAuth(this.getContext?.());
     }
+    /**
+     * Self-authorize now that the global gate is non-rejecting (memory-soul-
+     * read-gate family fix, ops-oox7 — applying the Memory.ts/Soul.ts pattern
+     * to WorkspaceState/Relationship/Integration/MemoryGrant). Closes the same
+     * P0 leak: Harper routes `GET /WorkspaceState/<id>` to get() and the
+     * collection describe (`GET /WorkspaceState`) to a path outside search(),
+     * so neither was gated before this fix — an anonymous caller got a 200 with
+     * full record content. Per-record ownership scoping happens in get() below;
+     * the collection scope is still in search().
+     */
+    allowRead() { return allowVerified(this.getContext?.()); }
+    /**
+     * Override get() to scope by-id reads the same way search() scopes
+     * collection reads (memory-soul-read-gate family fix). Never distinguishes
+     * "doesn't exist" from "exists but not yours" — both return 404, never
+     * 403, so a denied caller can't use get() to enumerate other agents'
+     * workspace-state ids.
+     */
+    async get(target) {
+        // Collection / query reads — the `GET /WorkspaceState/?<query>` form and
+        // the bare collection — arrive as a RequestTarget with `isCollection ===
+        // true`, and are governed by search() (same owner scoping). Only a
+        // genuine by-id get is ownership-checked below. Without this guard,
+        // get() would receive the query's RequestTarget, super.get() would
+        // return the (truthy) result set, and the single-record ownership check
+        // would find no `.agentId` on it (see Memory.ts's get() for the full
+        // rationale — same bug class).
+        if (!target || (typeof target === "object" && target.isCollection)) {
+            return this.search(target);
+        }
+        const auth = await this._auth();
+        // Anonymous by-id read is already blocked at the allowRead() gate (403);
+        // this is defense-in-depth if get() is ever reached directly.
+        if (auth.kind === "anonymous") {
+            return NOT_FOUND();
+        }
+        // Trusted internal call or admin agent — unfiltered, unchanged behavior.
+        if (auth.kind === "internal" || (auth.kind === "agent" && auth.isAdmin)) {
+            return super.get(target);
+        }
+        // Non-admin agent: only its own workspace-state records.
+        const record = await super.get(target);
+        if (!record)
+            return NOT_FOUND();
+        if (record.agentId !== auth.agentId)
+            return NOT_FOUND();
+        return record;
+    }
     /**
      * Override search() to scope collection GETs to the authenticated agent's own
      * records. Internal calls + admin agents see all; anonymous is denied
@@ -85,7 +134,12 @@ export class WorkspaceState extends databases.flair.WorkspaceState {
         if (auth.kind === "internal" || (auth.kind === "agent" && auth.isAdmin)) {
             return super.delete(id);
         }
-        const record = await this.get(id);
+        // Use super.get(id), NOT this.get(id): the new get() override above 404s
+        // (a truthy Response) for a non-owner id, which would otherwise defeat
+        // the `if (!record)` check below and mis-route a genuinely-missing
+        // record into the FORBIDDEN branch instead of a clean super.delete(id)
+        // no-op. Mirrors Memory.ts's delete() — same rationale, same fix.
+        const record = await super.get(id);
         if (!record)
             return super.delete(id);
         if (record.agentId !== auth.agentId) {

package/dist/resources/dedup.js

@@ -0,0 +1,62 @@
+// ─── Conservative near-duplicate detection primitives (Harper-free) ─────────
+// Memory-integrity fix (#526/#548 field regressions). This module is
+// deliberately Harper-free — same rationale as ./bm25.ts and ./scoring.ts —
+// so the cosine+Jaccard co-gate math is unit-testable directly against the
+// SHIPPED code without a live Harper. The DB-touching top-1-cosine lookup
+// (needs `databases`) lives in ./Memory.ts, built on top of these primitives.
+//
+// ── THE INVARIANT ────────────────────────────────────────────────────────────
+// This module NEVER decides whether to write. It only scores a candidate
+// match. Callers (Memory.post / Memory.put) always write the new record; a
+// "conservative match" is surfaced to the caller as a SIGNAL on the response
+// (deduplicated/matchedId/matchConfidence), never as a reason to skip the
+// write. That was the #526 bug: two topically-close but DISTINCT findings —
+// one about replication route-directionality, one about DDL/schema
+// replication — and the second was silently dropped because raw cosine alone
+// flagged it as a "duplicate" of the first. Requiring BOTH cosine AND lexical
+// (Jaccard token-overlap) to clear their thresholds against the SAME single
+// top-cosine candidate catches true near-dups while sparing topic collisions
+// (high cosine, low lexical) — and even then, only ever as a non-suppressing
+// signal.
+import { tokenize } from "./bm25.js";
+/** Default raw-cosine similarity threshold for a candidate to even be considered. */
+export const DEDUP_COSINE_THRESHOLD_DEFAULT = 0.95;
+/** Default Jaccard token-overlap threshold (of the top cosine candidate). */
+export const DEDUP_LEXICAL_THRESHOLD_DEFAULT = 0.5;
+/** Below this content length, similarity scoring is unreliable ("ok" would
+ *  match "ok" trivially) — the dedup gate is bypassed entirely. */
+export const DEDUP_MIN_CONTENT_LENGTH = 20;
+/** Jaccard similarity of two token sets: |A∩B| / |A∪B|. An empty side yields 0
+ *  (no signal — never treated as "fully similar" by vacuous set equality). */
+export function jaccardSimilarity(tokensA, tokensB) {
+    const a = new Set(tokensA);
+    const b = new Set(tokensB);
+    if (a.size === 0 || b.size === 0)
+        return 0;
+    let intersection = 0;
+    for (const t of a)
+        if (b.has(t))
+            intersection++;
+    const union = a.size + b.size - intersection;
+    return union === 0 ? 0 : intersection / union;
+}
+/**
+ * Conservative match = cosine AND lexical BOTH clear their thresholds.
+ * Both gates are required — a topic collision (high cosine, low lexical: two
+ * DIFFERENT findings that merely share a topic/vocabulary) must NOT be
+ * flagged as a duplicate. Only a true near-duplicate (high cosine AND high
+ * lexical overlap) is flagged.
+ */
+export function isConservativeMatch(cosine, lexical, cosineThreshold = DEDUP_COSINE_THRESHOLD_DEFAULT, lexicalThreshold = DEDUP_LEXICAL_THRESHOLD_DEFAULT) {
+    return cosine >= cosineThreshold && lexical >= lexicalThreshold;
+}
+/** Compute the {cosine, lexical} confidence pair for a candidate against the
+ *  new content's text, using the shared bm25 tokenizer. Rounded to 3dp to
+ *  match the existing _score rounding convention (SemanticSearch.ts). */
+export function computeMatchConfidence(newContent, candidateContent, cosine) {
+    const lexical = jaccardSimilarity(tokenize(newContent), tokenize(candidateContent || ""));
+    return {
+        cosine: Math.round(cosine * 1000) / 1000,
+        lexical: Math.round(lexical * 1000) / 1000,
+    };
+}

package/dist/resources/health.js

@@ -2,6 +2,7 @@ import { Resource, databases } from "@harperfast/harper";
 import { promises as fsp } from "node:fs";
 import { homedir, platform } from "node:os";
 import { join } from "node:path";
+import { getRerankStatus } from "./rerank-provider.js";
 const db = databases;
 const redactHome = (p) => {
     const home = homedir();
@@ -515,6 +516,22 @@ export class HealthDetail extends Resource {
         catch {
             stats.bridges = null;
         }
+        // ── Reranker ──
+        // Cross-encoder rerank stage. Reports flag/model/mode + live counters so we
+        // can see if it's silently degrading (fallbackCount climbing) in prod.
+        try {
+            const rr = getRerankStatus();
+            stats.rerank = rr;
+            if (rr.enabled && rr.state === "failed") {
+                warnings.push({ level: "warn", message: `reranker enabled but unavailable: ${rr.error ?? "init failed"} — recall falling back to vector order` });
+            }
+            if (rr.enabled && rr.rerankCount > 0 && rr.fallbackCount > rr.rerankCount) {
+                warnings.push({ level: "warn", message: `reranker falling back more than reranking (${rr.fallbackCount} fallbacks / ${rr.rerankCount} reranks) — check latency budget / topN` });
+            }
+        }
+        catch {
+            stats.rerank = null;
+        }
         // ── Warnings ──
         stats.warnings = warnings;
         // ── Process info ──

package/dist/resources/mcp-tools.js

@@ -1,17 +1,17 @@
 /**
- * mcp-tools.ts — the 9 curated flair tools for the Model-2 custom /mcp handler.
+ * mcp-tools.ts — the 10 curated flair tools for the Model-2 custom /mcp handler.
  *
- * Curated BY CONSTRUCTION: this module implements exactly the 9 tools that the
+ * Curated BY CONSTRUCTION: this module implements exactly the 10 tools that the
  * `@tpsdev-ai/flair-mcp` stdio proxy exposes, each a thin wrapper over the
  * existing flair Resource handler. No business logic is re-implemented — the
  * wrapped handlers (Memory / SemanticSearch / BootstrapMemories / Soul /
  * WorkspaceState / OrgEvent) enforce per-agent scoping/ownership via
  * `resolveAgentAuth(getContext())`, so the MCP surface inherits the SAME security
  * model as the signed-REST path. There is no raw CRUD surface — the only way to
- * reach the datastore through /mcp is via one of these 9 semantic tools.
+ * reach the datastore through /mcp is via one of these 10 semantic tools.
  *
- *   memory_search · memory_store · memory_get · memory_delete · bootstrap ·
- *   soul_set · soul_get · flair_workspace_set · flair_orgevent
+ *   memory_search · memory_store · memory_update · memory_get · memory_delete ·
+ *   bootstrap · soul_set · soul_get · flair_workspace_set · flair_orgevent
  *
  * ── The scoping seam ────────────────────────────────────────────────────────
  * The /mcp handler resolves the OAuth token's `sub` → a flair `Agent` id, then
@@ -116,6 +116,58 @@ async function memoryStore(agent, args) {
         tags: args?.tags,
     }));
 }
+/**
+ * memory_update — id-targeted, dedup-BYPASSED overwrite/version path (memory-
+ * integrity fix). Mirrors flair-client's MemoryApi.update() (packages/
+ * flair-client/src/client.ts), reimplemented against the resource instance
+ * API instead of HTTP since this handler calls the Memory resource directly
+ * (same pattern as memoryStore vs the flair-mcp stdio tool). Auth is enforced
+ * by Memory.get()/Memory.put()/Memory.post()'s EXISTING ownership checks — no
+ * parallel auth logic here.
+ *
+ * Default (preserveHistory unset/false): read the existing record, merge the
+ * new content on top (Harper PUT is full-record replacement — never send a
+ * bare partial), clear the stale embedding so the server regenerates it, and
+ * PUT the merged record back to the SAME id.
+ *
+ * preserveHistory: true: write a NEW id with `supersedes: id`. Memory.post()
+ * validates/authorizes the supersede (denying a cross-agent supersede without
+ * a "write" MemoryGrant) and closes the old record's validTo AFTER the new
+ * record is written (never the reverse — see resources/Memory.ts).
+ */
+async function memoryUpdate(agent, args) {
+    const Cls = await handler("Memory");
+    const h = new Cls(undefined, delegationContext(agent));
+    const id = args?.id;
+    const content = args?.content;
+    const preserveHistory = args?.preserveHistory === true;
+    const existing = await h.get(id);
+    if (!existing) {
+        return { error: "memory not found", status: 404 };
+    }
+    if (preserveHistory) {
+        const newId = `${agent.agentId}-${crypto.randomUUID()}`;
+        const record = {
+            ...existing,
+            id: newId,
+            content,
+            supersedes: id,
+            createdAt: new Date().toISOString(),
+        };
+        delete record.updatedAt;
+        delete record.embedding;
+        delete record.embeddingModel;
+        delete record.validFrom;
+        delete record.validTo;
+        delete record.archivedAt;
+        h.isCollection = true;
+        return unwrap(await h.post(record));
+    }
+    const merged = { ...existing, content, updatedAt: new Date().toISOString() };
+    delete merged.embedding;
+    delete merged.embeddingModel;
+    return unwrap(await h.put(merged));
+}
 async function memoryGet(agent, args) {
     const Cls = await handler("Memory");
     const h = new Cls(undefined, delegationContext(agent));
@@ -231,6 +283,24 @@ export const TOOLS = {
         },
         impl: memoryStore,
     },
+    memory_update: {
+        def: {
+            name: "memory_update",
+            description: "Update an existing memory by ID. Dedup-bypassed (this is an intentional overwrite, not a new write). " +
+                "Default: overwrites the same id in place. Pass preserveHistory=true to instead write a new version " +
+                "linked via `supersedes`, closing the old one's validity window.",
+            inputSchema: {
+                type: "object",
+                properties: {
+                    id: { type: "string", description: "ID of the memory to update" },
+                    content: { type: "string", description: "New content" },
+                    preserveHistory: { type: "boolean", description: "Write a new version (supersedes-linked) instead of overwriting in place (default false)" },
+                },
+                required: ["id", "content"],
+            },
+        },
+        impl: memoryUpdate,
+    },
     memory_get: {
         def: {
             name: "memory_get",