@tpsdev-ai/flair 0.51.1 → 0.52.0

package/dist/resources/SemanticSearch.js

@@ -1,7 +1,9 @@
 import { Resource, databases } from "harper";
 import { resolveAgentAuth, allowVerified } from "./agent-auth.js";
 import { getEmbedding, getMode } from "./embeddings-provider.js";
-import { patchRecord, withDetachedTxn } from "./table-helpers.js";
+import { isEmbeddingSpaceUniform, spaceGuardDiagnostics } from "./embedding-space-guard.js";
+import { withDetachedTxn } from "./table-helpers.js";
+import { applyHitStats, noteSearchHits } from "./hit-tracking.js";
 import { checkRateLimit, rateLimitResponse } from "./rate-limiter.js";
 import { resolveReadScope } from "./memory-read-scope.js";
 // The BM25 + union-RRF hybrid path is feature-flagged via hybridEnabled()
@@ -61,7 +63,7 @@ export class SemanticSearch extends Resource {
         // recall-harness (test/bench/recall-harness/run.ts) and `recall-eval.mjs`
         // before reconsidering this default if the compositeScore formula or
         // corpus changes.
-        const { agentId: bodyAgentId, q, queryEmbedding, tag, subject, subjects, limit = 10, includeSuperseded = false, scoring = "raw", minScore = 0, since, asOf, includeTrust = false, includeMetadata = false, abstain = false, explain = false, includeLegs = false, includeArchived = false } = data || {};
+        const { agentId: bodyAgentId, q, queryEmbedding, tag, subject, subjects, limit = 10, includeSuperseded = false, scoring = "raw", minScore = 0, since, asOf, includeTrust = false, includeMetadata = false, includeTrigger = false, abstain = false, explain = false, includeLegs = false, includeArchived = false } = data || {};
         // Authenticated identity lives on the Harper Resource context (getContext().request).
         // `this.request` is NOT populated on Harper v5 Resources — prior reads here
         // silently returned undefined and the defense-in-depth scope check below
@@ -121,6 +123,24 @@ export class SemanticSearch extends Resource {
             }
             catch { }
         }
+        // ─── Vector-space uniformity guard (embedding-space-guard slice 1) ────────
+        // If the store is NOT uniform in the current embedding space (a mixed-space
+        // corpus during a re-embed / model change), cosining the query against
+        // stored vectors returns garbage — Harper zero-pads a mismatched-dimension
+        // vector and returns a bogus score instead of throwing; same-dims /
+        // different-space is silent garbage too. REFUSE the embedding leg and
+        // degrade to keyword-only LOUDLY: drop qEmb so retrieveCandidates takes the
+        // keyword-only branch (reusing the existing mode:'none' graceful-degrade
+        // contract), and surface a structured warning naming both spaces + the
+        // `flair reembed` remedy below. Never serve mixed-space vectors, never
+        // merely log. Consulted through the SAME single chokepoint the write-time
+        // dedup leg uses (resources/embedding-space-guard.ts) — O(1) on the uniform
+        // happy path.
+        let spaceGuardDegraded = false;
+        if (qEmb && !(await isEmbeddingSpaceUniform())) {
+            qEmb = undefined;
+            spaceGuardDegraded = true;
+        }
         // ─── Temporal intent detection ────────────────────────────────────────────
         let sinceDate = since ? new Date(since) : null;
         let temporalBoost = 1.0;
@@ -250,10 +270,17 @@ export class SemanticSearch extends Resource {
             // with `includeMetadata: true`. `subject` needs no widening — it is
             // already in DEFAULT_SELECT. Neither flag ⇒ select stays undefined ⇒
             // response bytes unchanged.
-            select: (includeTrust || includeMetadata)
+            // flair#1546: `includeTrigger` opts the skill recall signal into the
+            // projection. DEFAULT_SELECT deliberately omits `trigger` (a skill-only
+            // column — the shared retrieval core must not grow it for every consumer,
+            // same K&S projection ruling as `metadata`), so skill_search opts it in
+            // per-request to return the trigger in its lightweight catalog. Neither
+            // flag ⇒ select stays undefined ⇒ response bytes unchanged.
+            select: (includeTrust || includeMetadata || includeTrigger)
                 ? [...DEFAULT_SELECT,
                     ...(includeTrust ? ["provenance"] : []),
-                    ...(includeMetadata ? ["metadata"] : [])]
+                    ...(includeMetadata ? ["metadata"] : []),
+                    ...(includeTrigger ? ["trigger"] : [])]
                 : undefined,
             // flair#744 slice 2 + confidence-band refinement: attach the absolute
             // per-result cosine confidence when the caller opts into abstention OR
@@ -291,15 +318,13 @@ export class SemanticSearch extends Resource {
         // the final slice needs no additional sort. A cross-encoder rerank stage
         // used to sit here and reorder the pool before this slice; it was removed
         // in flair#893 after measuring Δp@3 = 0.000 at 4.1× query latency.
-        const topResults = filteredResults.slice(0, limit);
-        // Async hit tracking — don't block the response
+        // Overlay committed hit stats BEFORE noting this search's increment so
+        // the response still shows the pre-hit count (same contract as the old
+        // fire-and-forget Memory patch, which ran after the slice).
+        const topResults = await Promise.all(filteredResults.slice(0, limit).map((r) => applyHitStats(r, ctx)));
+        // Async hit tracking — MemoryHitStat only, never a Memory rewrite.
         const now = new Date().toISOString();
-        for (const r of topResults) {
-            patchRecord(databases.flair.Memory, r.id, {
-                retrievalCount: (r.retrievalCount || 0) + 1,
-                lastRetrieved: now,
-            }).catch(() => { });
-        }
+        noteSearchHits(topResults.map((r) => r.id), now, ctx);
         // flair#744 slice 1 — opt-in inline trust-evidence block. Assembled HERE,
         // in the response tail, strictly AFTER read-scope resolution
         // (retrieveCandidates + scope.isAllowed already ran) and purely for the
@@ -334,7 +359,14 @@ export class SemanticSearch extends Resource {
             response.bestScore = abstention.bestScore;
             response.threshold = abstention.threshold;
         }
-        if (!qEmb && q && getMode() === "none") {
+        if (spaceGuardDegraded) {
+            const diag = spaceGuardDiagnostics();
+            response._warning =
+                `semantic search unavailable — the store has mixed embedding spaces ` +
+                    `(current: ${diag.current}; found: ${diag.found.join(", ")}); results are keyword-only. ` +
+                    `Reconcile to one space with: flair reembed --stale-only`;
+        }
+        else if (!qEmb && q && getMode() === "none") {
             response._warning = "semantic search unavailable — results are keyword-only";
         }
         // flair#1358: opt-in per-leg candidate ids for the bench instrument.

package/dist/resources/Soul.js

@@ -1,25 +1,21 @@
 import { databases } from "harper";
-import { resolveAgentAuth } from "./agent-auth.js";
+import { guardOwnerFieldImmutable } from "./owner-field-guard.js";
 import { localInstanceId } from "./instance-identity.js";
-import { makeAuthGate, stampAttribution, UNAUTH } from "./record-type-kit.js";
+import { makeAuthGate, stampAttribution } from "./record-type-kit.js";
 import { RECORD_TYPES } from "./record-types.js";
-/**
- * Deny anonymous; enforce per-agent write ownership for non-admin agents.
- * The previous header-based check only fired when an agent WAS present (it read
- * x-tps-agent), so an anonymous request — which carries no x-tps-agent — slipped
- * through. With the non-rejecting gate, each write path self-enforces (resolveAgentAuth
- * distinguishes internal/agent/anonymous). Mirrors the WorkspaceState pattern.
- *
- * No-forge attribution — mode/field drawn from RECORD_TYPES.Soul (record-
- * types slice 2, flair#520) rather than hand-typed literals. "validate-
- * truthy" (see record-type-kit.ts's stampAttribution doc) — rejects a
- * PRESENT, mismatched agentId; passes through untouched when absent. Same
- * idiom as Memory.post()/put().
- */
+import { authorizeSoulWrite, refuseSoulWriteContent, soulProvenance } from "./soul-write-policy.js";
+// Source authorization is independent of principal ownership: an admin runtime
+// may manage records elsewhere, but it cannot author identity-defining Soul.
 async function enforceWriteAuth(self, data) {
-    const auth = await resolveAgentAuth(self.getContext?.());
-    if (auth.kind === "anonymous")
-        return UNAUTH();
+    const { auth, source, denied } = await authorizeSoulWrite(self.getContext?.());
+    if (denied)
+        return denied;
+    if (!data || typeof data !== "object" || Array.isArray(data)) {
+        return new Response(JSON.stringify({ error: "soul_write_requires_one_record" }), {
+            status: 400, headers: { "Content-Type": "application/json" },
+        });
+    }
+    data.provenance = soulProvenance(auth, source, new Date().toISOString());
     const attr = stampAttribution(auth, data, RECORD_TYPES.Soul.ownerField, RECORD_TYPES.Soul.attribution.post, "forbidden: agentId must match authenticated agent");
     return attr.denied ?? null;
 }
@@ -43,6 +39,10 @@ export class Soul extends databases.flair.Soul {
         const denied = await enforceWriteAuth(this, content);
         if (denied)
             return denied;
+        // Learned artifacts cannot gain identity authority through an operator write.
+        const learnedDenied = await refuseSoulWriteContent(content);
+        if (learnedDenied)
+            return learnedDenied;
         content.durability ||= "permanent";
         content.createdAt = new Date().toISOString();
         content.updatedAt = content.createdAt;
@@ -56,10 +56,40 @@ export class Soul extends databases.flair.Soul {
         }
         return super.post(content, context);
     }
+    // PATCH must validate the merged value and retained legacy tags, rather
+    // than treating an omitted field as proof that no learned content exists.
+    async patch(content, query) {
+        const denied = await enforceWriteAuth(this, content);
+        if (denied)
+            return denied;
+        const denial = await guardOwnerFieldImmutable(this, () => super.get(), content, "agentId");
+        if (denial)
+            return denial;
+        // Fail-closed, same as Memory's stored-state read: a throw aborts the
+        // write; missing/unreadable stored state cannot authorize a PATCH that
+        // typically omits agentId (that used to skip the content-provenance match).
+        const existing = await super.get();
+        if (!existing || typeof existing !== "object" || existing instanceof Response) {
+            return new Response(JSON.stringify({ error: "soul_stored_state_unavailable" }), {
+                status: 403,
+                headers: { "Content-Type": "application/json" },
+            });
+        }
+        const learnedDenied = await refuseSoulWriteContent({ ...existing, ...content });
+        if (learnedDenied)
+            return learnedDenied;
+        return super.patch(content, query);
+    }
     async put(content, context) {
         const denied = await enforceWriteAuth(this, content);
         if (denied)
             return denied;
+        const learnedDenied = await refuseSoulWriteContent(content);
+        if (learnedDenied)
+            return learnedDenied;
+        const ownerDenial = await guardOwnerFieldImmutable(this, () => super.get(), content, "agentId");
+        if (ownerDenial)
+            return ownerDenial;
         content.updatedAt = new Date().toISOString();
         // Write-time originatorInstanceId stamp — see post() above / Memory.ts's
         // stampOriginatorInstanceId doc. No-op if already set.
@@ -68,4 +98,10 @@ export class Soul extends databases.flair.Soul {
         }
         return super.put(content, context);
     }
+    async delete(id) {
+        const { denied } = await authorizeSoulWrite(this.getContext?.());
+        if (denied)
+            return denied;
+        return super.delete(id);
+    }
 }

package/dist/resources/WorkspaceState.js

@@ -9,6 +9,7 @@
  */
 import { databases } from "harper";
 import { resolveAgentAuth } from "./agent-auth.js";
+import { guardOwnerFieldImmutable } from "./owner-field-guard.js";
 import { invalidEntitiesResponse } from "./entity-vocab.js";
 import { makeAuthGate, makeReadScope, makeByIdReadGate, makeScopedSearch, resolveAuthGate, stampAttribution, FORBIDDEN, UNAUTH, } from "./record-type-kit.js";
 import { RECORD_TYPES } from "./record-types.js";
@@ -115,7 +116,18 @@ export class WorkspaceState extends databases.flair.WorkspaceState {
             return entitiesError;
         return super.post(content);
     }
+    // PATCH routes past put(), so agentId immutability is enforced on both verbs
+    // via the one shared delegate.
+    async patch(content, query) {
+        const denial = await guardOwnerFieldImmutable(this, () => super.get(), content, "agentId");
+        if (denial)
+            return denial;
+        return super.patch(content, query);
+    }
     async put(content) {
+        const __ownerDenial = await guardOwnerFieldImmutable(this, () => super.get(), content, "agentId");
+        if (__ownerDenial)
+            return __ownerDenial;
         const auth = await this._auth();
         if (auth.kind === "anonymous")
             return UNAUTH();

package/dist/resources/auth-middleware.js

@@ -444,6 +444,11 @@ server.http(async (request, nextLayer) => {
             (method === "POST" || method === "PUT" || method === "PATCH")) {
             if (!request.tpsAgentIsAdmin) {
                 try {
+                    // NOTE: dead code — Harper's middleware Request has no parsed body
+                    // (no .clone()/.json()), so this body-check never fires. Owner/
+                    // attribution enforcement lives in the resource layer (record-owner-
+                    // guard + owner-field-guard). Kept for a separate cleanup PR — not
+                    // live coverage.
                     const clone = request.clone();
                     const body = await clone.json();
                     if (body?.authorId && body.authorId !== agentId) {
@@ -478,6 +483,11 @@ server.http(async (request, nextLayer) => {
             (method === "POST" || method === "PUT" || method === "PATCH")) {
             if (!request.tpsAgentIsAdmin) {
                 try {
+                    // NOTE: dead code — Harper's middleware Request has no parsed body
+                    // (no .clone()/.json()), so this body-check never fires. Owner/
+                    // attribution enforcement lives in the resource layer (record-owner-
+                    // guard + owner-field-guard). Kept for a separate cleanup PR — not
+                    // live coverage.
                     const clone = request.clone();
                     const body = await clone.json();
                     if (body?.agentId && body.agentId !== agentId) {
@@ -546,6 +556,11 @@ server.http(async (request, nextLayer) => {
                 if (method !== "DELETE") {
                     let bodyAgentId = null;
                     try {
+                        // NOTE: dead code — Harper's middleware Request has no parsed body
+                        // (no .clone()/.json()), so this body-check never fires. Owner/
+                        // attribution enforcement lives in the resource layer (record-owner-
+                        // guard + owner-field-guard). Kept for a separate cleanup PR — not
+                        // live coverage.
                         const clone = request.clone();
                         const body = await clone.json();
                         bodyAgentId = body?.agentId ?? null;
@@ -562,50 +577,8 @@ server.http(async (request, nextLayer) => {
                 // one condition enforced in two places is how the two drift apart.
             }
         }
-        // Memory promotion guard: only admin can approve or set durability=permanent
-        if (((url.pathname === "/Memory" || url.pathname.startsWith("/Memory/") || url.pathname === "/memory" || url.pathname.startsWith("/memory/"))) &&
-            (method === "PUT" || method === "POST" || method === "PATCH")) {
-            if (!request.tpsAgentIsAdmin) {
-                try {
-                    const clone = request.clone();
-                    const body = await clone.json();
-                    const setsApproved = body?.promotionStatus === "approved";
-                    const setsPermanent = body?.durability === "permanent";
-                    const setsArchived = body?.archived === true;
-                    if (setsApproved || setsPermanent || setsArchived) {
-                        return new Response(JSON.stringify({
-                            error: "forbidden: only admins can approve promotions, set permanent durability, or archive memories"
-                        }), { status: 403 });
-                    }
-                }
-                catch { }
-            }
-        }
-        // Memory DELETE: permanent memories are admin-only to purge.
-        //
-        // The OWNERSHIP half of this guard — which was the model the shared
-        // record-ownership rule above was built from, and the only one in this file
-        // that already covered PATCH — now lives there and covers every table. What
-        // remains here is the part that is NOT about ownership: durability. An agent
-        // owning a permanent memory still may not purge it.
-        if (((url.pathname === "/Memory" || url.pathname.startsWith("/Memory/") || url.pathname === "/memory" || url.pathname.startsWith("/memory/"))) &&
-            method === "DELETE") {
-            if (!request.tpsAgentIsAdmin) {
-                try {
-                    const pathParts = url.pathname.split("/").filter(Boolean);
-                    const memId = pathParts[1] ? decodeURIComponent(pathParts[1]) : null;
-                    if (memId) {
-                        const record = await databases.flair.Memory.get(memId);
-                        if (record?.durability === "permanent") {
-                            return new Response(JSON.stringify({
-                                error: "forbidden: only admins can purge permanent memories"
-                            }), { status: 403 });
-                        }
-                    }
-                }
-                catch { }
-            }
-        }
+        // Memory workflow-field provenance is enforced on parsed resource writes.
+        // Memory deletion uses the shared stored-owner rule for every tier.
     }
     // ── WorkspaceState read guard: agent-scoped reads ───────────────────────────
     if (method === "GET" && !request.tpsAgentIsAdmin) {

package/dist/resources/authority-field-guard.js

@@ -0,0 +1,37 @@
+/** Workflow verdicts can only be stamped by their trusted raw-table paths.
+ * Keep unchanged echoes compatible with full-row clients, and preserve omitted
+ * stamps on PUT; omission must not erase a verdict. A content change drops the
+ * stamp: the verdict is bound to the reviewed text. No admin/body bypass. */
+export const AUTHORITY_FIELDS = {
+    Memory: ["promotionStatus", "promotedAt", "promotedBy"],
+};
+export async function guardAuthorityFields(getExisting, content, table) {
+    const existing = await getExisting();
+    for (const field of AUTHORITY_FIELDS[table]) {
+        if (Object.hasOwn(content, field) && content[field] !== existing?.[field]) {
+            return new Response(JSON.stringify({ error: `forbidden: ${field} is set by the promotion workflow` }), {
+                status: 403, headers: { "content-type": "application/json" },
+            });
+        }
+    }
+    // The stamp means this specific content was reviewed. Echo-tolerance and
+    // omitted-field restore apply only when the body does not change `content`.
+    // Null the columns (do not merely delete) so PATCH and PUT both drop them.
+    if (existing && Object.hasOwn(content, "content") && content.content !== existing.content) {
+        for (const field of AUTHORITY_FIELDS[table])
+            content[field] = null;
+        return null;
+    }
+    for (const field of AUTHORITY_FIELDS[table]) {
+        if (existing && Object.hasOwn(existing, field))
+            content[field] = existing[field];
+    }
+    return null;
+}
+/** Drop workflow stamps from a request-origin raw write. Unconditional: even
+ * stamps the guard would restore onto an omitted-field update must not land
+ * through ingest paths that are not a promotion-stamp site. */
+export function stripAuthorityFields(record, table) {
+    for (const field of AUTHORITY_FIELDS[table])
+        delete record[field];
+}

package/dist/resources/bm25-index-service.js

@@ -127,7 +127,7 @@ function record(ev) {
 }
 /** Read-your-write hook: call immediately after a committed Memory write that
  *  changed content or any scope/temporal attribute. Safe to call for writes
- *  that changed neither (it is an idempotent re-index of one row). */
+ *  that changed neither (an unchanged indexed projection is ignored). */
 export function noteMemoryUpsert(row) {
     const r = project(row);
     if (r)

package/dist/resources/bm25-index.js

@@ -89,7 +89,21 @@ export const SUPPORTED_SCOPE_ATTRS = [
 export const TEMPORAL_ATTRS = ["createdAt", "expiresAt", "validFrom", "validTo"];
 /** The `select` a corpus scan needs in order to feed this index. */
 export const INDEX_SELECT = ["id", "content", ...SUPPORTED_SCOPE_ATTRS, ...TEMPORAL_ATTRS];
+// Bound exact-body retention independently of corpus size. Oversized bodies
+// and evicted entries use the existing re-tokenization path, never a hash.
+const MAX_CACHED_BODIES = 1024;
+const MAX_CACHED_BODY_CHARS = 512 * 1024;
+const INDEX_META_ATTRS = [...SUPPORTED_SCOPE_ATTRS, ...TEMPORAL_ATTRS];
 const SUPPORTED_ATTR_SET = new Set(SUPPORTED_SCOPE_ATTRS);
+// Indexed metadata consists of scalars and arrays of scalars. Unknown object
+// shapes never qualify for the fast path. Arrays are copied on admission so a
+// caller reusing and mutating its record cannot change the comparison snapshot.
+function sameIndexValue(a, b) {
+    if (Array.isArray(a) && Array.isArray(b)) {
+        return a.length === b.length && a.every((v, i) => (v === null || typeof v !== "object") && v === b[i]);
+    }
+    return (a === null || typeof a !== "object") && a === b;
+}
 function emptyPosting() {
     return { slots: new Int32Array(4), tfs: new Int32Array(4), len: 0 };
 }
@@ -181,12 +195,33 @@ export class Bm25Index {
     sweptTo = 0;
     /** Slots retired by the sweep — removed from the aggregates. */
     retired = new Set();
+    cachedBodies = new Map();
+    cachedBodyChars = 0;
+    forgetBody(id) {
+        const content = this.cachedBodies.get(id);
+        if (content === undefined)
+            return;
+        this.cachedBodyChars -= content.length;
+        this.cachedBodies.delete(id);
+    }
+    rememberBody(id, content) {
+        this.forgetBody(id);
+        if (content.length > MAX_CACHED_BODY_CHARS)
+            return;
+        while (this.cachedBodies.size >= MAX_CACHED_BODIES || this.cachedBodyChars + content.length > MAX_CACHED_BODY_CHARS) {
+            this.forgetBody(this.cachedBodies.keys().next().value);
+        }
+        this.cachedBodies.set(id, content);
+        this.cachedBodyChars += content.length;
+    }
     get size() { return this.slotOf.size; }
     /** Live postings, for the memory-footprint assertions in the tests. */
     get postingCount() { return this.totalPostings - this.deadPostings; }
     get termCount() { return this.postings.size; }
     has(id) { return this.slotOf.has(id); }
     clear() {
+        this.cachedBodies.clear();
+        this.cachedBodyChars = 0;
         this.slots = [];
         this.slotOf.clear();
         this.freeSlots = [];
@@ -201,30 +236,32 @@ export class Bm25Index {
         this.retired.clear();
     }
     // ─── Maintenance ──────────────────────────────────────────────────────────
-    /**
-     * Add or replace a document. Deliberately NOT "diff the content and patch
-     * the postings": an upsert always tombstones the old slot and appends a new
-     * one. Detecting an unchanged body would need a content fingerprint, and a
-     * fingerprint collision is a silently-wrong lexical index — the one failure
-     * mode this index may not have. The cost is a dead posting run per update,
-     * reclaimed by `compact()` below at O(1) amortized.
-     */
+    /** Ignore writes whose complete indexed projection is unchanged. Relevant
+     *  changes still replace the slot, preserving aggregate and expiry handling. */
     upsert(record) {
         const id = record?.id;
         if (typeof id !== "string" || id.length === 0)
             return;
+        const content = record.content || "";
+        const previousSlot = this.slotOf.get(id);
+        const previous = previousSlot === undefined ? null : this.slots[previousSlot];
+        if (previous && this.cachedBodies.get(id) === content &&
+            INDEX_META_ATTRS.every((attr) => (attr in previous.meta) === (attr in record) && sameIndexValue(previous.meta[attr], record[attr]))) {
+            this.rememberBody(id, content);
+            return;
+        }
         this.remove(id);
-        const tokens = tokenize(record.content || "");
+        const tokens = tokenize(content);
         const tf = new Map();
         for (const t of tokens)
             tf.set(t, (tf.get(t) || 0) + 1);
         const meta = { id };
         for (const attr of SUPPORTED_SCOPE_ATTRS)
             if (attr in record)
-                meta[attr] = record[attr];
+                meta[attr] = Array.isArray(record[attr]) ? record[attr].slice() : record[attr];
         for (const attr of TEMPORAL_ATTRS)
             if (attr in record)
-                meta[attr] = record[attr];
+                meta[attr] = Array.isArray(record[attr]) ? record[attr].slice() : record[attr];
         const slot = this.freeSlots.length > 0 ? this.freeSlots.pop() : this.slots.length;
         const entry = {
             id,
@@ -236,6 +273,7 @@ export class Bm25Index {
         };
         this.slots[slot] = entry;
         this.slotOf.set(id, slot);
+        this.rememberBody(id, content);
         for (const [term, count] of tf) {
             let p = this.postings.get(term);
             if (!p) {
@@ -262,6 +300,7 @@ export class Bm25Index {
             return;
         const entry = this.slots[slot];
         this.slotOf.delete(id);
+        this.forgetBody(id);
         this.slots[slot] = null;
         if (entry) {
             if (!this.retired.delete(slot))