@tpsdev-ai/flair 0.5.1 → 0.5.3

package/dist/cli.js

@@ -7,7 +7,28 @@ import { join, resolve as resolvePath } from "node:path";
 import { spawn } from "node:child_process";
 import { createPrivateKey, sign as nodeCryptoSign, randomUUID } from "node:crypto";
 import { keystore } from "./keystore.js";
-import { signBody } from "../resources/federation-crypto.js";
+// Federation crypto helpers — inlined to avoid cross-boundary imports from
+// src/ into resources/, which don't survive npm packaging (see also
+// resources/federation-crypto.ts; the two must stay in sync).
+function sortKeys(val) {
+    if (val === null || val === undefined || typeof val !== "object")
+        return val;
+    if (Array.isArray(val))
+        return val.map(sortKeys);
+    const sorted = {};
+    for (const key of Object.keys(val).sort()) {
+        sorted[key] = sortKeys(val[key]);
+    }
+    return sorted;
+}
+function canonicalize(obj) {
+    return JSON.stringify(sortKeys(obj));
+}
+function signBody(body, secretKey) {
+    const message = new TextEncoder().encode(canonicalize(body));
+    const sig = nacl.sign.detached(message, secretKey);
+    return Buffer.from(sig).toString("base64url");
+}
 // ─── Defaults ────────────────────────────────────────────────────────────────
 const DEFAULT_PORT = 19926;
 const DEFAULT_OPS_PORT = 19925;

package/dist/resources/Memory.js

@@ -1,5 +1,5 @@
 import { databases } from "@harperfast/harper";
-import { patchRecord } from "./table-helpers.js";
+import { patchRecord, withDetachedTxn } from "./table-helpers.js";
 import { isAdmin } from "./auth-middleware.js";
 import { getEmbedding, getModelId } from "./embeddings-provider.js";
 import { scanContent, isStrictMode } from "./content-safety.js";
@@ -41,20 +41,31 @@ export class Memory extends databases.flair.Memory {
         const agentIdCondition = allowedOwners.length === 1
             ? { attribute: "agentId", comparator: "equals", value: allowedOwners[0] }
             : { conditions: allowedOwners.map(id => ({ attribute: "agentId", comparator: "equals", value: id })), operator: "or" };
-        // Harper passes `query` as a RequestTarget (extends URLSearchParams) with pathname, id, etc.
-        // Table.search() reads `target.conditions` from it. We inject our scope condition there.
+        // Harper passes `query` as a RequestTarget (extends URLSearchParams). Table.search()
+        // reads `target.conditions`; when that is unset, it iterates URLSearchParams entries
+        // as conditions instead. We inject our scope condition into `.conditions`, which
+        // means URL params (except agentId, which we always enforce) must also be translated
+        // to conditions or they'll be silently dropped.
         if (query && typeof query === "object" && !Array.isArray(query)) {
-            const existing = query.conditions ?? [];
-            query.conditions = Array.isArray(existing)
-                ? [agentIdCondition, ...existing]
-                : [agentIdCondition, existing];
-            return super.search(query);
+            const existing = [];
+            if (Array.isArray(query.conditions) && query.conditions.length > 0) {
+                existing.push(...query.conditions);
+            }
+            else if (typeof query.entries === "function") {
+                for (const [k, v] of query.entries()) {
+                    if (k === "agentId")
+                        continue;
+                    existing.push({ attribute: k, comparator: "equals", value: v });
+                }
+            }
+            query.conditions = [agentIdCondition, ...existing];
+            return withDetachedTxn(ctx, () => super.search(query));
         }
         // Fallback: plain array or no query (internal calls)
         const conditions = Array.isArray(query) && query.length > 0
             ? [agentIdCondition, ...query]
             : [agentIdCondition];
-        return super.search(conditions);
+        return withDetachedTxn(ctx, () => super.search(conditions));
     }
     async post(content, context) {
         // Rate limiting — use authenticated agent ID, not client-supplied body field
@@ -128,6 +139,25 @@ export class Memory extends databases.flair.Memory {
         return super.post(content);
     }
     async put(content) {
+        // Reindex migration bypass: admin-only escape hatch used by the
+        // MemoryReindex admin endpoint to re-PUT each existing record byte-for-byte
+        // (no updatedAt bump, no embedding regen, no safety rescan) so Harper
+        // repopulates secondary indices. Because this skips content safety and
+        // auditability, it must be gated to admins. Internal calls (no auth
+        // context) pass through, matching the pattern used in delete().
+        if (content._reindex === true) {
+            const ctx = this.getContext?.();
+            const request = ctx?.request ?? ctx;
+            const actorId = request?.tpsAgent;
+            if (actorId && !(await isAdmin(actorId))) {
+                return new Response(JSON.stringify({ error: "reindex_admin_only" }), {
+                    status: 403,
+                    headers: { "Content-Type": "application/json" },
+                });
+            }
+            delete content._reindex;
+            return super.put(content);
+        }
         const now = new Date().toISOString();
         content.updatedAt = now;
         // Set defaults that post() sets — put() is also used for new records via CLI

package/dist/resources/MemoryReindex.js

@@ -0,0 +1,129 @@
+/**
+ * MemoryReindex.ts — One-shot migration: heal Harper secondary indices on Memory.
+ *
+ * POST /MemoryReindex — admin-only.
+ *
+ * Body:
+ *   { dryRun?: boolean, agentId?: string, batchSize?: number }
+ *
+ * Behaviour:
+ *   Scans the Memory primary store (unfiltered, via the base table class) and
+ *   compares per-agent primary counts against per-agent secondary-index counts
+ *   (an equals lookup on agentId). Any agent whose secondary count is below
+ *   its primary count has unindexed records. Re-PUT every record with the
+ *   _reindex escape hatch so Memory.put() preserves every field byte-for-byte
+ *   (no updatedAt bump, no embedding regen, no safety rescan).
+ *
+ * Why this exists: Harper's background runIndexing() pass populates secondary
+ * indices only on schema changes that add a new indexed attribute. If that
+ * pass is interrupted, or if a schema version change doesn't re-register an
+ * existing index, older records live in the primary store but never make it
+ * into the secondary index. Scoped search() via RequestTarget.conditions —
+ * which uses the agentId index — then returns empty for those agents.
+ *
+ * This endpoint is idempotent: running it again finds zero drift.
+ */
+import { Resource, databases } from "@harperfast/harper";
+import { isAdmin } from "./auth-middleware.js";
+export class MemoryReindex extends Resource {
+    async post(data) {
+        const ctx = this.getContext?.();
+        const request = ctx?.request ?? ctx ?? this.request;
+        const authAgent = request?.tpsAgent;
+        const basicAgent = request?.headers?.get?.("x-tps-agent");
+        const isBasicAdmin = basicAgent === "admin";
+        const isEd25519Admin = Boolean(authAgent) && request?.tpsAgentIsAdmin === true;
+        if (!isBasicAdmin && !(isEd25519Admin && (await isAdmin(authAgent)))) {
+            return new Response(JSON.stringify({ error: "forbidden: admin required" }), { status: 403, headers: { "Content-Type": "application/json" } });
+        }
+        const dryRun = data?.dryRun === true;
+        const agentFilter = typeof data?.agentId === "string" ? data.agentId : null;
+        const batchSize = Math.max(1, Math.min(Number(data?.batchSize) || 100, 1000));
+        const Memory = databases.flair.Memory;
+        const stats = {
+            scanned: 0,
+            agentsWithDrift: 0,
+            totalMissing: 0,
+            reindexed: 0,
+            errors: 0,
+            dryRun,
+            agentFilter,
+            drift: [],
+            errorSamples: [],
+        };
+        // Pass 1: primary-store scan. Count records per agent.
+        const primaryByAgent = new Map();
+        const recordsToReindex = [];
+        for await (const record of Memory.search()) {
+            if (agentFilter && record.agentId !== agentFilter)
+                continue;
+            if (!record.id || !record.agentId)
+                continue;
+            primaryByAgent.set(record.agentId, (primaryByAgent.get(record.agentId) ?? 0) + 1);
+            recordsToReindex.push(record.id);
+        }
+        stats.scanned = recordsToReindex.length;
+        // Pass 2: for each agent, probe the secondary index via an agentId-only equals
+        // lookup. Harper's query planner can't reorder a single condition, so this
+        // actually hits Table.indices.agentId. Drift = primary - indexed.
+        for (const [agentId, primary] of primaryByAgent) {
+            let indexed = 0;
+            try {
+                for await (const _ of Memory.search({
+                    conditions: [{ attribute: "agentId", comparator: "equals", value: agentId }],
+                })) {
+                    indexed++;
+                }
+            }
+            catch (err) {
+                stats.errors++;
+                if (stats.errorSamples.length < 5) {
+                    stats.errorSamples.push({ id: agentId, message: err?.message ?? String(err) });
+                }
+                continue;
+            }
+            const missing = primary - indexed;
+            if (missing > 0) {
+                stats.agentsWithDrift++;
+                stats.totalMissing += missing;
+                stats.drift.push({ agentId, primary, indexed, missing });
+            }
+        }
+        stats.drift.sort((a, b) => b.missing - a.missing);
+        if (dryRun) {
+            return {
+                message: `dry run: ${stats.totalMissing} record${stats.totalMissing === 1 ? "" : "s"} missing from agentId index across ${stats.agentsWithDrift} agent${stats.agentsWithDrift === 1 ? "" : "s"}. Reindex will re-PUT all ${stats.scanned} records to heal.`,
+                stats,
+            };
+        }
+        // Pass 3: re-PUT every primary-store record with _reindex=true so Memory.put()
+        // preserves every field byte-for-byte. The re-PUT forces Harper to re-insert
+        // into all secondary indices. Cheaper-than-sound variants (only re-PUT records
+        // that appear missing) would miss rows the primary scan sees fine but that
+        // Harper's read path can't locate via any index path.
+        for (let i = 0; i < recordsToReindex.length; i += batchSize) {
+            const chunk = recordsToReindex.slice(i, i + batchSize);
+            for (const id of chunk) {
+                try {
+                    const record = await Memory.get(id);
+                    if (!record) {
+                        stats.errors++;
+                        continue;
+                    }
+                    await Memory.put({ ...record, _reindex: true });
+                    stats.reindexed++;
+                }
+                catch (err) {
+                    stats.errors++;
+                    if (stats.errorSamples.length < 5) {
+                        stats.errorSamples.push({ id, message: err?.message ?? String(err) });
+                    }
+                }
+            }
+        }
+        return {
+            message: `reindex complete: ${stats.reindexed} re-indexed, ${stats.errors} errors, ${stats.totalMissing} pre-existing index gap${stats.totalMissing === 1 ? "" : "s"} healed`,
+            stats,
+        };
+    }
+}

package/dist/resources/SemanticSearch.js

@@ -1,6 +1,6 @@
 import { Resource, databases } from "@harperfast/harper";
 import { getEmbedding, getMode } from "./embeddings-provider.js";
-import { patchRecord } from "./table-helpers.js";
+import { patchRecord, withDetachedTxn } from "./table-helpers.js";
 import { checkRateLimit, rateLimitResponse } from "./rate-limiter.js";
 import { wrapUntrusted } from "./content-safety.js";
 // ─── Temporal Decay + Relevance Scoring ─────────────────────────────────────
@@ -178,7 +178,11 @@ export class SemanticSearch extends Resource {
             if (conditions.length > 0) {
                 query.conditions = conditions;
             }
-            for await (const record of databases.flair.Memory.search(query)) {
+            // MemoryGrant.search above left a closed transaction in ctx's chain —
+            // detach it so Harper builds a fresh transaction for this Memory read.
+            const ctx = this.getContext?.();
+            const memoryResults = withDetachedTxn(ctx, () => databases.flair.Memory.search(query));
+            for await (const record of memoryResults) {
                 if (record.expiresAt && Date.parse(record.expiresAt) < Date.now())
                     continue;
                 if (sinceDate && record.createdAt && new Date(record.createdAt) < sinceDate)
@@ -214,7 +218,11 @@ export class SemanticSearch extends Resource {
             // Full scan is only used when there's no query embedding (e.g. tag-only
             // or subject-only searches, or when the embedding engine is unavailable).
             const query = conditions.length > 0 ? { conditions } : {};
-            for await (const record of databases.flair.Memory.search(query)) {
+            // MemoryGrant.search above left a closed transaction in ctx's chain —
+            // detach it so Harper builds a fresh transaction for this Memory read.
+            const ctx = this.getContext?.();
+            const memoryResults = withDetachedTxn(ctx, () => databases.flair.Memory.search(query));
+            for await (const record of memoryResults) {
                 if (record.expiresAt && Date.parse(record.expiresAt) < Date.now())
                     continue;
                 if (sinceDate && record.createdAt && new Date(record.createdAt) < sinceDate)

package/dist/resources/table-helpers.js

@@ -28,8 +28,33 @@ export function patchRecordSilent(table, id, patch) {
 // Harper put() = FULL RECORD REPLACEMENT. Missing fields are deleted permanently.
 // Always use patchRecord() or patchRecordSilent().
 // ─────────────────────────────────────────────────────────────────────────────
-// ── RULE ──────────────────────────────────────────────────────────────────────
-// Never call `tables.X.put(partial)` directly anywhere in Flair resources.
-// Harper put() = FULL RECORD REPLACEMENT. Missing fields are deleted permanently.
-// Always use patchRecord() or patchRecordSilent().
-// ─────────────────────────────────────────────────────────────────────────────
+/**
+ * Detach ctx.transaction while `fn` runs, then restore.
+ *
+ * Why: when a request does two table reads in sequence (e.g. MemoryGrant.search
+ * then Memory.search), the first generator drains and leaves its transaction
+ * CLOSED at the tail of ctx.transaction's linked chain. Harper's txnForContext
+ * (Table.js ~3633) walks that chain when opening a transaction for the second
+ * table and inherits the CLOSED state onto the fresh transaction — which then
+ * silently reads zero rows.
+ *
+ * Clearing ctx.transaction forces Harper to build a brand-new ImmediateTransaction
+ * for the inner call. Table.search captures that transaction synchronously into
+ * its result generator's closure, so the saved chain can be restored immediately
+ * without affecting the streaming read.
+ *
+ * Use this whenever a resource method reads one table, then reads another in
+ * the same request context.
+ */
+export function withDetachedTxn(ctx, fn) {
+    if (!ctx)
+        return fn();
+    const saved = ctx.transaction;
+    ctx.transaction = undefined;
+    try {
+        return fn();
+    }
+    finally {
+        ctx.transaction = saved;
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tpsdev-ai/flair",
-  "version": "0.5.1",
+  "version": "0.5.3",
   "description": "Identity, memory, and soul for AI agents. Cryptographic identity (Ed25519), semantic memory with local embeddings, and persistent personality — all in a single process.",
   "type": "module",
   "license": "Apache-2.0",
@@ -36,6 +36,8 @@
     "SECURITY.md"
   ],
   "scripts": {
+    "clean": "node -e \"require('fs').rmSync('dist',{recursive:true,force:true})\"",
+    "prebuild": "npm run clean",
     "build": "tsc -p tsconfig.json --noCheck",
     "build:cli": "tsc -p tsconfig.cli.json --noCheck",
     "prepublishOnly": "npm run build && npm run build:cli",

package/dist/resources/embeddings.js

@@ -1,42 +0,0 @@
-/**
- * Fallback hash-based embedding (used when sidecar is unavailable).
- * Real embeddings come from the embed-server sidecar (harper-fabric-embeddings).
- */
-const DIMS = 512;
-function h1(s) { let h = 5381; for (let i = 0; i < s.length; i++)
-    h = ((h << 5) + h + s.charCodeAt(i)) >>> 0; return h % DIMS; }
-function h2(s) { let h = 0x811c9dc5; for (let i = 0; i < s.length; i++) {
-    h ^= s.charCodeAt(i);
-    h = Math.imul(h, 0x01000193) >>> 0;
-} return h % DIMS; }
-export function fallbackEmbed(text) {
-    const tokens = text.toLowerCase().replace(/[^a-z0-9\s-]/g, ' ').split(/\s+/).filter(t => t.length > 1);
-    const clean = text.toLowerCase().replace(/\s+/g, ' ');
-    const vec = new Float64Array(DIMS);
-    for (const t of tokens) {
-        vec[h1(t)] += 2;
-        vec[h2(t)] += 1;
-    }
-    for (let i = 0; i < tokens.length - 1; i++)
-        vec[h1(tokens[i] + '_' + tokens[i + 1])] += 1.5;
-    for (let i = 0; i <= clean.length - 3; i++)
-        vec[h1(clean.slice(i, i + 3))] += 0.5;
-    for (let i = 0; i < DIMS; i++)
-        if (vec[i] > 0)
-            vec[i] = 1 + Math.log(vec[i]);
-    let norm = 0;
-    for (let i = 0; i < DIMS; i++)
-        norm += vec[i] * vec[i];
-    norm = Math.sqrt(norm);
-    if (norm > 0)
-        for (let i = 0; i < DIMS; i++)
-            vec[i] /= norm;
-    return Array.from(vec);
-}
-export function cosineSimilarity(a, b) {
-    let dot = 0;
-    const len = Math.min(a.length, b.length);
-    for (let i = 0; i < len; i++)
-        dot += a[i] * b[i];
-    return dot;
-}