@tpsdev-ai/flair 0.20.1 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -11
- package/SECURITY.md +52 -20
- package/config.yaml +32 -5
- package/dist/cli.js +2235 -260
- package/dist/deploy.js +96 -15
- package/dist/doctor-client.js +312 -0
- package/dist/fleet-presence.js +98 -0
- package/dist/fleet-verify.js +291 -0
- package/dist/install/clients.js +18 -0
- package/dist/mcp-client-assertion.js +364 -0
- package/dist/probe.js +97 -0
- package/dist/rem/restore.js +1 -1
- package/dist/rem/runner.js +82 -12
- package/dist/resources/Admin.js +1 -1
- package/dist/resources/AdminConnectors.js +1 -1
- package/dist/resources/AdminDashboard.js +1 -1
- package/dist/resources/AdminIdp.js +1 -1
- package/dist/resources/AdminInstance.js +1 -1
- package/dist/resources/AdminMemory.js +2 -2
- package/dist/resources/AdminPrincipals.js +1 -1
- package/dist/resources/Agent.js +14 -0
- package/dist/resources/AgentCard.js +1 -1
- package/dist/resources/AgentSeed.js +1 -1
- package/dist/resources/AttentionQuery.js +356 -0
- package/dist/resources/Credential.js +11 -1
- package/dist/resources/Federation.js +121 -2
- package/dist/resources/Integration.js +1 -1
- package/dist/resources/MCPClientMetadata.js +88 -0
- package/dist/resources/Memory.js +127 -22
- package/dist/resources/MemoryBootstrap.js +441 -85
- package/dist/resources/MemoryGrant.js +1 -1
- package/dist/resources/MemoryReflect.js +105 -49
- package/dist/resources/MemoryUsage.js +104 -0
- package/dist/resources/OAuth.js +69 -5
- package/dist/resources/OrgEvent.js +11 -0
- package/dist/resources/OrgEventCatchup.js +1 -1
- package/dist/resources/Presence.js +268 -17
- package/dist/resources/RecordUsage.js +230 -0
- package/dist/resources/Relationship.js +108 -26
- package/dist/resources/SemanticSearch.js +88 -320
- package/dist/resources/SkillScan.js +15 -0
- package/dist/resources/Soul.js +14 -0
- package/dist/resources/WorkspaceLatest.js +1 -1
- package/dist/resources/WorkspaceState.js +12 -1
- package/dist/resources/agent-auth.js +61 -3
- package/dist/resources/agentcard-fields.js +2 -2
- package/dist/resources/auth-middleware.js +42 -14
- package/dist/resources/bm25-filter.js +1 -1
- package/dist/resources/bm25.js +13 -7
- package/dist/resources/collision-lib.js +157 -0
- package/dist/resources/dedup.js +2 -2
- package/dist/resources/ed25519-auth.js +2 -2
- package/dist/resources/embeddings-boot.js +149 -0
- package/dist/resources/embeddings-provider.js +364 -106
- package/dist/resources/entity-vocab.js +139 -0
- package/dist/resources/federation-nonce-store.js +195 -0
- package/dist/resources/health.js +97 -6
- package/dist/resources/instance-identity.js +53 -0
- package/dist/resources/mcp-client-metadata-fields.js +112 -0
- package/dist/resources/mcp-handler.js +1 -1
- package/dist/resources/mcp-tools.js +88 -10
- package/dist/resources/memory-bootstrap-lib.js +1 -1
- package/dist/resources/memory-read-scope.js +58 -71
- package/dist/resources/memory-reflect-lib.js +289 -0
- package/dist/resources/memory-visibility.js +37 -0
- package/dist/resources/migration-boot.js +143 -0
- package/dist/resources/migrations/dir-safety.js +71 -0
- package/dist/resources/migrations/embedding-stamp.js +178 -0
- package/dist/resources/migrations/envelope.js +43 -0
- package/dist/resources/migrations/export.js +38 -0
- package/dist/resources/migrations/ledger.js +55 -0
- package/dist/resources/migrations/lock.js +154 -0
- package/dist/resources/migrations/progress.js +31 -0
- package/dist/resources/migrations/registry.js +36 -0
- package/dist/resources/migrations/risk-policy.js +23 -0
- package/dist/resources/migrations/runner.js +456 -0
- package/dist/resources/migrations/snapshot.js +127 -0
- package/dist/resources/migrations/source-fields.js +93 -0
- package/dist/resources/migrations/space.js +73 -0
- package/dist/resources/migrations/state.js +64 -0
- package/dist/resources/migrations/status.js +39 -0
- package/dist/resources/migrations/synthetic-test-migration.js +93 -0
- package/dist/resources/migrations/types.js +9 -0
- package/dist/resources/presence-internal.js +29 -0
- package/dist/resources/provenance.js +50 -0
- package/dist/resources/rate-limiter.js +8 -1
- package/dist/resources/scoring.js +124 -5
- package/dist/resources/semantic-retrieval-core.js +317 -0
- package/dist/version-check.js +167 -0
- package/dist/version-handshake.js +122 -0
- package/package.json +4 -5
- package/schemas/agent.graphql +7 -0
- package/schemas/event.graphql +5 -0
- package/schemas/federation.graphql +12 -0
- package/schemas/memory.graphql +82 -0
- package/schemas/schema.graphql +5 -43
- package/schemas/workspace.graphql +3 -0
- package/dist/resources/IngestEvents.js +0 -162
- package/dist/resources/IssueTokens.js +0 -19
- package/dist/resources/ObsAgentSnapshot.js +0 -13
- package/dist/resources/ObsEventFeed.js +0 -13
- package/dist/resources/ObsOffice.js +0 -19
- package/dist/resources/ObservationCenter.js +0 -23
- package/ui/observation-center.html +0 -385
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* space.ts — pre-flight space check, step 1 of the ladder
|
|
3
|
+
* (flair#695, "Space pressure"):
|
|
4
|
+
*
|
|
5
|
+
* "Pre-flight space check before any write — estimate snapshot size AND the
|
|
6
|
+
* migration's own working set (supersession transforms temporarily hold
|
|
7
|
+
* old+new ≈ 2× for touched tables) vs free disk, with a headroom floor
|
|
8
|
+
* (never fill past ~90% — a full disk bricks Harper itself)."
|
|
9
|
+
*
|
|
10
|
+
* Kern verdict: "Space estimate includes HNSW rebuild overhead (re-embed
|
|
11
|
+
* rewrites every vector → index rebuild can be 2–3× raw vector size):
|
|
12
|
+
* snapshot + new vectors + index overhead vs headroom floor." —
|
|
13
|
+
* `estimateWorkingSetBytes` below is the caller's job (the runner computes
|
|
14
|
+
* it per risk class); this module just evaluates the estimate against real
|
|
15
|
+
* (or test-overridden) disk free/total space.
|
|
16
|
+
*/
|
|
17
|
+
import { statfsSync } from "node:fs";
|
|
18
|
+
export const DEFAULT_HEADROOM_FLOOR = 0.9; // never project past 90% used
|
|
19
|
+
/**
|
|
20
|
+
* Test-only escape hatch: when set, overrides the real statfs-reported free
|
|
21
|
+
* byte count. Used by the halt-on-blocked-space integration test (real
|
|
22
|
+
* Harper spawned as a child process — the test can't inject a JS dependency
|
|
23
|
+
* into that process, only environment variables) to force the ladder's
|
|
24
|
+
* step-1 check to fail deterministically without needing an actually-full
|
|
25
|
+
* disk. Unset (the default) in every real deployment, so this is inert in
|
|
26
|
+
* production.
|
|
27
|
+
*/
|
|
28
|
+
export const TEST_FREE_BYTES_ENV = "FLAIR_MIGRATION_TEST_FREE_BYTES";
|
|
29
|
+
function realGetFreeBytes(dataDir) {
|
|
30
|
+
const override = process.env[TEST_FREE_BYTES_ENV];
|
|
31
|
+
if (override !== undefined) {
|
|
32
|
+
const n = Number(override);
|
|
33
|
+
if (Number.isFinite(n) && n >= 0)
|
|
34
|
+
return n;
|
|
35
|
+
}
|
|
36
|
+
const st = statfsSync(dataDir);
|
|
37
|
+
return st.bavail * st.bsize;
|
|
38
|
+
}
|
|
39
|
+
function realGetTotalBytes(dataDir) {
|
|
40
|
+
const st = statfsSync(dataDir);
|
|
41
|
+
return st.blocks * st.bsize;
|
|
42
|
+
}
|
|
43
|
+
export const defaultSpaceProbe = {
|
|
44
|
+
getFreeBytes: realGetFreeBytes,
|
|
45
|
+
getTotalBytes: realGetTotalBytes,
|
|
46
|
+
};
|
|
47
|
+
/**
|
|
48
|
+
* Evaluates the pre-flight space check. Fails (ok:false) if the needed
|
|
49
|
+
* bytes don't fit in free space OR if consuming them would project disk
|
|
50
|
+
* usage past the headroom floor — matching invariant III's "never fill past
|
|
51
|
+
* ~90%" rule exactly (a technically-fitting migration that would still push
|
|
52
|
+
* the disk to 95% full is refused, same as one that doesn't fit at all).
|
|
53
|
+
*/
|
|
54
|
+
export function checkSpace(input, probe = defaultSpaceProbe) {
|
|
55
|
+
const floor = input.headroomFloor ?? DEFAULT_HEADROOM_FLOOR;
|
|
56
|
+
const freeBytes = probe.getFreeBytes(input.dataDir);
|
|
57
|
+
const totalBytes = probe.getTotalBytes(input.dataDir);
|
|
58
|
+
const neededBytes = input.estimatedSnapshotBytes + input.estimatedWorkingSetBytes;
|
|
59
|
+
const usedBytesNow = Math.max(0, totalBytes - freeBytes);
|
|
60
|
+
const projectedUsedBytes = usedBytesNow + neededBytes;
|
|
61
|
+
const projectedUsedFraction = totalBytes > 0 ? projectedUsedBytes / totalBytes : 1;
|
|
62
|
+
const fits = neededBytes <= freeBytes;
|
|
63
|
+
const withinFloor = projectedUsedFraction <= floor;
|
|
64
|
+
const ok = fits && withinFloor;
|
|
65
|
+
let reason;
|
|
66
|
+
if (!ok) {
|
|
67
|
+
reason =
|
|
68
|
+
`need ${neededBytes} bytes free (snapshot + migration working set), have ${freeBytes}; ` +
|
|
69
|
+
`proceeding would exceed the ${Math.round(floor * 100)}% disk headroom floor — ` +
|
|
70
|
+
`prune old snapshots (flair keeps last-3/30-day) or set FLAIR_SNAPSHOT_DIR to a volume with more room`;
|
|
71
|
+
}
|
|
72
|
+
return { ok, freeBytes, totalBytes, neededBytes, projectedUsedFraction, reason };
|
|
73
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* state.ts — the health-tracked "last migration completed at version X"
|
|
3
|
+
* marker (Kern verdict, detect() fallback): "detect() cheap + read-only —
|
|
4
|
+
* bounded query (limit=1 style), never O(corpus) on boot; answers 'is there
|
|
5
|
+
* work,' not 'how much.' Fallback: a health-tracked 'last migration
|
|
6
|
+
* completed at version X' short-circuit."
|
|
7
|
+
*
|
|
8
|
+
* A small on-disk JSON sidecar (same idiom as the REM nightly log / PID
|
|
9
|
+
* file / version-check cache — plain files under the instance's own
|
|
10
|
+
* footprint, not a Harper table: adding a new table to record migration
|
|
11
|
+
* state would itself be a migration, a chicken-and-egg this avoids
|
|
12
|
+
* entirely). Lives under `<dataDir>/.migrations/state.json` — co-located
|
|
13
|
+
* with the data it protects rather than under a fixed `~/.flair`, since
|
|
14
|
+
* HDB_ROOT can point anywhere per-instance (matches resources/health.ts's
|
|
15
|
+
* own dataDir resolution).
|
|
16
|
+
*
|
|
17
|
+
* Once a migration's entry shows `completedAtVersion === <the currently
|
|
18
|
+
* running version>` and `lastOutcome === "success"`, the runner skips
|
|
19
|
+
* calling that migration's detect() ENTIRELY on this and every subsequent
|
|
20
|
+
* boot at the same version — zero query cost, not just a cheap one.
|
|
21
|
+
*/
|
|
22
|
+
import { existsSync, readFileSync, statSync } from "node:fs";
|
|
23
|
+
import { dirname, join } from "node:path";
|
|
24
|
+
import { ensureSecureDir, writeSecureFile } from "./dir-safety.js";
|
|
25
|
+
export function defaultStatePath(dataDir) {
|
|
26
|
+
return join(dataDir, ".migrations", "state.json");
|
|
27
|
+
}
|
|
28
|
+
export function readMigrationState(path) {
|
|
29
|
+
try {
|
|
30
|
+
if (!existsSync(path))
|
|
31
|
+
return {};
|
|
32
|
+
const raw = JSON.parse(readFileSync(path, "utf-8"));
|
|
33
|
+
return raw && typeof raw === "object" ? raw : {};
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
// Corrupt/unreadable state — treat as "nothing known yet", never throw.
|
|
37
|
+
// Worst case this costs one extra (cheap, bounded) detect() call.
|
|
38
|
+
return {};
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
export function writeMigrationStateEntry(path, id, entry) {
|
|
42
|
+
const dir = dirname(path);
|
|
43
|
+
ensureSecureDir(dir);
|
|
44
|
+
const current = readMigrationState(path);
|
|
45
|
+
current[id] = entry;
|
|
46
|
+
writeSecureFile(path, JSON.stringify(current, null, 2) + "\n", dir);
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* True when this migration can be skipped WITHOUT calling detect() at all —
|
|
50
|
+
* it already completed successfully at the currently running version.
|
|
51
|
+
*/
|
|
52
|
+
export function isShortCircuited(state, migrationId, runningVersion) {
|
|
53
|
+
const entry = state[migrationId];
|
|
54
|
+
return !!entry && entry.lastOutcome === "success" && entry.completedAtVersion === runningVersion;
|
|
55
|
+
}
|
|
56
|
+
/** For health/doctor: the most recent state-file mtime, if any (cheap "when did anything last change" signal). */
|
|
57
|
+
export function statePathMtime(path) {
|
|
58
|
+
try {
|
|
59
|
+
return statSync(path).mtime.toISOString();
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* status.ts — composes the in-process progress store (progress.ts) with the
|
|
3
|
+
* on-disk state file (state.ts) into the single snapshot resources/health.ts
|
|
4
|
+
* and `flair doctor` (via /HealthDetail) surface.
|
|
5
|
+
*
|
|
6
|
+
* Falls back to the on-disk state when the in-process progress map is empty
|
|
7
|
+
* — a process can be asked for /HealthDetail before its boot's `setImmediate`
|
|
8
|
+
* callback has fired at all (a very early poll), or after a restart where
|
|
9
|
+
* this run's cycle hasn't started yet; the on-disk record of the LAST
|
|
10
|
+
* completed/halted outcome is still genuinely useful in that window instead
|
|
11
|
+
* of reporting nothing.
|
|
12
|
+
*/
|
|
13
|
+
import { listMigrationProgress, getCycleStatus } from "./progress.js";
|
|
14
|
+
import { readMigrationState, defaultStatePath } from "./state.js";
|
|
15
|
+
export function getMigrationStatusSnapshot(dataDir) {
|
|
16
|
+
const cycle = getCycleStatus();
|
|
17
|
+
let migrations = listMigrationProgress();
|
|
18
|
+
if (migrations.length === 0) {
|
|
19
|
+
const state = readMigrationState(defaultStatePath(dataDir));
|
|
20
|
+
migrations = Object.entries(state).map(([id, entry]) => {
|
|
21
|
+
const derivedState = entry.lastOutcome === "success" ? "completed" : entry.lastOutcome === "halted" ? "halted" : "failed";
|
|
22
|
+
const row = {
|
|
23
|
+
id,
|
|
24
|
+
rowsDone: entry.rowsProcessed ?? 0,
|
|
25
|
+
rowsRemaining: entry.rowsRemaining ?? 0,
|
|
26
|
+
state: derivedState,
|
|
27
|
+
};
|
|
28
|
+
if (entry.reason)
|
|
29
|
+
row.reason = entry.reason;
|
|
30
|
+
return row;
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
return {
|
|
34
|
+
cyclePhase: cycle.phase,
|
|
35
|
+
lastCycleError: cycle.lastCycleError,
|
|
36
|
+
lastCycleAt: cycle.lastCycleAt,
|
|
37
|
+
migrations,
|
|
38
|
+
};
|
|
39
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* synthetic-test-migration.ts — the CI-only migration variant (K&S verdict):
|
|
3
|
+
* "Synthetic CI stamp variant MUST NOT ship — registration conditional on
|
|
4
|
+
* test/CI env; a prod user must never see a spurious ci-variant migration
|
|
5
|
+
* on first boot. Flip later = swap synthetic for real, same lanes/gates."
|
|
6
|
+
*
|
|
7
|
+
* This is the payload the test suite uses to exercise the FULL runner path
|
|
8
|
+
* end-to-end (detect → pre-flight → snapshot → batches → markers →
|
|
9
|
+
* completion gate → post-hash → ledger → prune) — riskClass
|
|
10
|
+
* 'schema-additive' (batch 50, full-envelope completion gate), distinct
|
|
11
|
+
* from embedding-stamp's 'derived-only' so the suite also covers the
|
|
12
|
+
* stricter envelope-comparison gate path via a real (ephemeral) Harper.
|
|
13
|
+
*
|
|
14
|
+
* SAFETY, belt-and-suspenders even though gating (shouldRegisterSyntheticMigration
|
|
15
|
+
* below) is what actually keeps this out of production:
|
|
16
|
+
* - Registration is conditional on FLAIR_ENABLE_TEST_MIGRATIONS === "1"
|
|
17
|
+
* (an explicit, exact-match opt-in — never a generic NODE_ENV=test
|
|
18
|
+
* check, which can be true in contexts nobody intended as "safe to run
|
|
19
|
+
* a schema-touching test migration").
|
|
20
|
+
* - Even IF it were somehow registered unconditionally, its detect/run
|
|
21
|
+
* queries are scoped to a single reserved, never-real agentId
|
|
22
|
+
* (RESERVED_TEST_AGENT_ID) — it can never see or touch a genuine
|
|
23
|
+
* agent's memories.
|
|
24
|
+
*
|
|
25
|
+
* Marker: stamps `Memory.source` (an existing, currently-unused-by-any-
|
|
26
|
+
* migration nullable String field — no new schema needed) to a value that
|
|
27
|
+
* encodes the migration id, so "is this row done" is answered entirely by
|
|
28
|
+
* re-reading the row itself (invariant IV: state lives IN the data).
|
|
29
|
+
*/
|
|
30
|
+
import { databases } from "@harperfast/harper";
|
|
31
|
+
export const SYNTHETIC_MIGRATION_ID = "synthetic-ci-schema-stamp";
|
|
32
|
+
export const RESERVED_TEST_AGENT_ID = "__flair_migration_synthetic_test_agent__";
|
|
33
|
+
export const SYNTHETIC_TARGET_MARKER = `${SYNTHETIC_MIGRATION_ID}-done`;
|
|
34
|
+
export const ENABLE_TEST_MIGRATIONS_ENV = "FLAIR_ENABLE_TEST_MIGRATIONS";
|
|
35
|
+
/**
|
|
36
|
+
* Exact-match opt-in — see the module doc. Exported so both the registry
|
|
37
|
+
* and a unit test can assert the exact condition, independent of the
|
|
38
|
+
* registry's own wiring.
|
|
39
|
+
*/
|
|
40
|
+
export function shouldRegisterSyntheticMigration(env = process.env) {
|
|
41
|
+
return env[ENABLE_TEST_MIGRATIONS_ENV] === "1";
|
|
42
|
+
}
|
|
43
|
+
function defaultMemoryTable() {
|
|
44
|
+
return databases.flair.Memory;
|
|
45
|
+
}
|
|
46
|
+
export function createSyntheticTestMigration(getTable = defaultMemoryTable) {
|
|
47
|
+
function pendingCondition() {
|
|
48
|
+
return [
|
|
49
|
+
{ attribute: "agentId", comparator: "equals", value: RESERVED_TEST_AGENT_ID },
|
|
50
|
+
{ attribute: "source", comparator: "not_equal", value: SYNTHETIC_TARGET_MARKER },
|
|
51
|
+
];
|
|
52
|
+
}
|
|
53
|
+
return {
|
|
54
|
+
id: SYNTHETIC_MIGRATION_ID,
|
|
55
|
+
riskClass: "schema-additive",
|
|
56
|
+
affectsTables: ["Memory"],
|
|
57
|
+
async detect() {
|
|
58
|
+
const table = getTable();
|
|
59
|
+
for await (const _row of table.search({ conditions: pendingCondition(), limit: 1 })) {
|
|
60
|
+
return true;
|
|
61
|
+
}
|
|
62
|
+
return false;
|
|
63
|
+
},
|
|
64
|
+
async countPending() {
|
|
65
|
+
const table = getTable();
|
|
66
|
+
let n = 0;
|
|
67
|
+
for await (const _row of table.search({ conditions: pendingCondition() }))
|
|
68
|
+
n++;
|
|
69
|
+
return n;
|
|
70
|
+
},
|
|
71
|
+
async run(batchSize) {
|
|
72
|
+
const table = getTable();
|
|
73
|
+
const candidates = [];
|
|
74
|
+
for await (const row of table.search({ conditions: pendingCondition(), limit: batchSize })) {
|
|
75
|
+
candidates.push(row);
|
|
76
|
+
}
|
|
77
|
+
const touchedIds = [];
|
|
78
|
+
for (const row of candidates) {
|
|
79
|
+
const id = String(row.id ?? "");
|
|
80
|
+
if (!id)
|
|
81
|
+
continue;
|
|
82
|
+
const existing = await table.get(id);
|
|
83
|
+
if (!existing)
|
|
84
|
+
continue;
|
|
85
|
+
if (existing.source === SYNTHETIC_TARGET_MARKER)
|
|
86
|
+
continue; // already stamped — idempotent skip
|
|
87
|
+
await table.put({ ...existing, source: SYNTHETIC_TARGET_MARKER });
|
|
88
|
+
touchedIds.push(id);
|
|
89
|
+
}
|
|
90
|
+
return { processed: touchedIds.length, touchedIds };
|
|
91
|
+
},
|
|
92
|
+
};
|
|
93
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* types.ts — shared shapes for the zero-touch migration runner (flair#695).
|
|
3
|
+
*
|
|
4
|
+
* Governed by flair#695 (the four invariants) and
|
|
5
|
+
* flair#695 (the runner design + K&S verdict
|
|
6
|
+
* refinements). Every migration registered in resources/migrations/registry.ts
|
|
7
|
+
* implements the `Migration` interface below.
|
|
8
|
+
*/
|
|
9
|
+
export {};
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
function presenceDelegationContext(auth) {
|
|
2
|
+
const agentAuth = auth.kind === "agent"
|
|
3
|
+
? { agentId: auth.agentId, isAdmin: auth.isAdmin }
|
|
4
|
+
: { agentId: "internal", isAdmin: true };
|
|
5
|
+
return { request: { _flairAgentAuth: agentAuth } };
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* The full presence roster (one row per agent — bounded, per the K&S
|
|
9
|
+
* verdict: "Presence is a bounded scan, one row/agent, tiny"), via the
|
|
10
|
+
* exported `Presence` resource's get(). Fails open to `[]` on any error
|
|
11
|
+
* (Presence being briefly unavailable should never break a caller that
|
|
12
|
+
* treats presence as one signal among several, not a hard dependency).
|
|
13
|
+
*/
|
|
14
|
+
export async function getPresenceRoster(auth) {
|
|
15
|
+
try {
|
|
16
|
+
const { Presence } = await import("./Presence.js");
|
|
17
|
+
const p = new Presence();
|
|
18
|
+
p.getContext = () => presenceDelegationContext(auth);
|
|
19
|
+
const roster = await p.get();
|
|
20
|
+
if (roster instanceof Response)
|
|
21
|
+
return [];
|
|
22
|
+
if (!Array.isArray(roster))
|
|
23
|
+
return [];
|
|
24
|
+
return roster;
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
return [];
|
|
28
|
+
}
|
|
29
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ─── Write-time provenance stamp (memory-provenance slice 1) ────────────────
|
|
3
|
+
*
|
|
4
|
+
* Foundational capture for an emergent-trust model: every write gets a
|
|
5
|
+
* structured, versioned `provenance` JSON blob recording what the server can
|
|
6
|
+
* actually VERIFY about the write, plus (optionally) what the caller merely
|
|
7
|
+
* CLAIMS. Deliberately minimal — verified fields only:
|
|
8
|
+
*
|
|
9
|
+
* { v: 1,
|
|
10
|
+
* verified: { agentId: <string|null>, timestamp: <ISO string> },
|
|
11
|
+
* claimed?: { model: <string> } }
|
|
12
|
+
*
|
|
13
|
+
* - `verified.agentId` comes from the ALREADY-RESOLVED auth verdict
|
|
14
|
+
* (resolveAgentAuth) — never from anything the caller can forge on the
|
|
15
|
+
* request body. `kind: "agent"` → the Ed25519-verified agentId. Any other
|
|
16
|
+
* verdict (in practice only `kind: "internal"` — a trusted in-process call
|
|
17
|
+
* with no per-agent identity to attribute) stamps `null` rather than
|
|
18
|
+
* throwing; `kind: "anonymous"` never reaches here — every write path
|
|
19
|
+
* already 401s it before this point.
|
|
20
|
+
* - `verified.timestamp` reuses the server-clock `createdAt` the caller has
|
|
21
|
+
* already computed by this point (never client-suppliable) — the same
|
|
22
|
+
* "stamp a dynamic attribute the server controls" mechanism as e.g. the
|
|
23
|
+
* `embeddingModel = getModelId()` stamp in resources/Memory.ts.
|
|
24
|
+
* - `claimed.model` is an OPTIONAL, UNVERIFIED passthrough: included only
|
|
25
|
+
* when the incoming write payload itself already carries a non-empty
|
|
26
|
+
* string `model` field. No client/CLI sets one today — this just means the
|
|
27
|
+
* server won't discard it if/when a future write path does. Never
|
|
28
|
+
* invented, never defaulted, and the `claimed` key is omitted entirely
|
|
29
|
+
* (not stamped as `{}`) when absent.
|
|
30
|
+
*
|
|
31
|
+
* Originally introduced in resources/Memory.ts (Memory.post()/Memory.put());
|
|
32
|
+
* extracted here so Relationship.ts (and any future write path) can reuse the
|
|
33
|
+
* EXACT same shape rather than inventing a table-specific format — the
|
|
34
|
+
* K&S-approved contract for the relationship-write-path spec is "reuse
|
|
35
|
+
* buildProvenance as-is," which this module makes literal (one function, one
|
|
36
|
+
* shape, imported by every writer) instead of a copy that could drift.
|
|
37
|
+
*/
|
|
38
|
+
export function buildProvenance(auth, createdAt, content) {
|
|
39
|
+
const provenance = {
|
|
40
|
+
v: 1,
|
|
41
|
+
verified: {
|
|
42
|
+
agentId: auth.kind === "agent" ? auth.agentId : null,
|
|
43
|
+
timestamp: createdAt,
|
|
44
|
+
},
|
|
45
|
+
};
|
|
46
|
+
if (typeof content?.model === "string" && content.model.length > 0) {
|
|
47
|
+
provenance.claimed = { model: content.model };
|
|
48
|
+
}
|
|
49
|
+
return JSON.stringify(provenance);
|
|
50
|
+
}
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
*
|
|
7
7
|
* FLAIR_RATE_LIMIT_RPM — requests per minute per agent (default: 120)
|
|
8
8
|
* FLAIR_RATE_LIMIT_EMBED — embedding requests per minute per agent (default: 30)
|
|
9
|
+
* FLAIR_RATE_LIMIT_USAGE — usage-feedback requests per minute per agent (default: 30)
|
|
9
10
|
* FLAIR_RATE_LIMIT_STORAGE — max memories per agent (default: 10000)
|
|
10
11
|
* FLAIR_RATE_LIMIT_ENABLED — "true" to enable (default: disabled for local, enabled when FLAIR_PUBLIC=true)
|
|
11
12
|
*/
|
|
@@ -35,7 +36,13 @@ export function checkRateLimit(agentId, bucket = "general") {
|
|
|
35
36
|
return { allowed: true }; // admin/internal calls
|
|
36
37
|
const limit = bucket === "embedding"
|
|
37
38
|
? getLimit("FLAIR_RATE_LIMIT_EMBED", 30)
|
|
38
|
-
:
|
|
39
|
+
// flair#683: usage-feedback is a write-that-affects-ranking (anti-gaming
|
|
40
|
+
// surface) — Sherlock's three-layer defense is rate-limiter (~30 RPM
|
|
41
|
+
// bucket) + per-agent dedup bound (MemoryUsage ledger) + the capped
|
|
42
|
+
// usageBoost itself. This bucket is that first layer.
|
|
43
|
+
: bucket === "usage"
|
|
44
|
+
? getLimit("FLAIR_RATE_LIMIT_USAGE", 30)
|
|
45
|
+
: getLimit("FLAIR_RATE_LIMIT_RPM", 120);
|
|
39
46
|
const key = `${agentId}:${bucket}`;
|
|
40
47
|
const now = Date.now();
|
|
41
48
|
const cutoff = now - WINDOW_MS;
|
|
@@ -40,13 +40,132 @@ export function retrievalBoost(retrievalCount) {
|
|
|
40
40
|
return 1.0;
|
|
41
41
|
return Math.min(1.0 + 0.1 * Math.log2(retrievalCount), RBOOST_CAP); // gentle, capped
|
|
42
42
|
}
|
|
43
|
+
// ─── usageBoost — the verified-USE signal (flair#683) ───────────────────────
|
|
44
|
+
// retrievalCount/retrievalBoost above count a search HIT as "used" — a
|
|
45
|
+
// rich-get-richer loop root-caused in #623 (a doc that surfaces once gets
|
|
46
|
+
// boosted, surfaces more, boosts more, independent of whether it was ever
|
|
47
|
+
// actually useful). usageCount is a STRONGER, distinct signal: it is only
|
|
48
|
+
// ever incremented by the dedicated usage-feedback endpoint (resources/
|
|
49
|
+
// MemoryUsage.ts) when a caller explicitly reports that a memory was
|
|
50
|
+
// cited/used to ground an answer or decision — never auto-incremented on
|
|
51
|
+
// search (schemas/memory.graphql's usageCount field doc). Dedup'd
|
|
52
|
+
// (agent, memory) ≤ 1 contribution and rate-limited there, so this function
|
|
53
|
+
// only ever sees a bounded, deliberately-reported count.
|
|
54
|
+
//
|
|
55
|
+
// SAME shape as retrievalBoost — a gentle, capped nudge (a tie-breaker, never
|
|
56
|
+
// an override), floor-gated so a popular-but-irrelevant doc gets no lift.
|
|
57
|
+
// K&S verdict (FLAIR-USAGE-FEEDBACK-SIGNAL.md, 2026-07-11): keep the
|
|
58
|
+
// constants/shape identical to retrievalBoost's so the ONLY variable in the
|
|
59
|
+
// harness rematch (test/bench/recall-harness) is signal QUALITY (what gets
|
|
60
|
+
// counted), not magnitude/shape — that isolation is what makes the
|
|
61
|
+
// positive/negative/noise comparison meaningful.
|
|
62
|
+
export const USAGE_BOOST_CAP = 1.1; // max +10% — a tie-breaker, not an override
|
|
63
|
+
export const USAGE_RELEVANCE_FLOOR = 0.5; // no boost at all for clearly-irrelevant docs
|
|
64
|
+
export function usageBoost(usageCount) {
|
|
65
|
+
if (!usageCount || usageCount <= 0)
|
|
66
|
+
return 1.0;
|
|
67
|
+
return Math.min(1.0 + 0.1 * Math.log2(usageCount), USAGE_BOOST_CAP); // gentle, capped
|
|
68
|
+
}
|
|
69
|
+
// flair#623 (2026-07-08): SemanticSearch.ts's `scoring` param now DEFAULTS to
|
|
70
|
+
// "raw" — compositeScore measurably HURTS recall-eval precision on the live
|
|
71
|
+
// corpus (Δp@3 -0.38 to -0.50 vs raw). recall-harness's harder 87-record
|
|
72
|
+
// corpus (test/bench/recall-harness) reproduced this in isolation at Δp@3
|
|
73
|
+
// -0.900: dWeight and rFactor applied UNCONDITIONALLY — no relevance-floor
|
|
74
|
+
// gate, unlike rBoost above — so an unrelated-but-`permanent`/fresh record
|
|
75
|
+
// (dWeight=1.0 × rFactor=1.0, no discount at all) routinely outranked the
|
|
76
|
+
// objectively best match once its `standard`/`persistent`, weeks-old
|
|
77
|
+
// durability/recency discount cut 30-95% off its score.
|
|
78
|
+
//
|
|
79
|
+
// FIRST ATTEMPT (kept here as a documented dead end — do not re-introduce):
|
|
80
|
+
// gate dWeight×rFactor by ramping it from neutral-at-floor to
|
|
81
|
+
// full-strength-at-rawScore=1.0, mirroring rBoost's floor gate. Measured
|
|
82
|
+
// WORSE than the original bug (p@3 0.033 vs 0.067) on recall-harness. Root
|
|
83
|
+
// cause: the records compositeScore actually needs to protect are the
|
|
84
|
+
// GENUINELY relevant ones, and a genuine match has a HIGH rawScore (0.9+ on
|
|
85
|
+
// this corpus, not near the 0.5 floor) — so ramping the gate open as
|
|
86
|
+
// rawScore rises means the *correct* answer gets the discount at nearly full
|
|
87
|
+
// strength (gate≈1), while the discount was already a no-op for the
|
|
88
|
+
// magnet distractors (permanent/fresh ⇒ dWeight×rFactor=1.0 regardless of any
|
|
89
|
+
// gate). Ramping-by-relevance is backwards for a DISCOUNT-only multiplier: it
|
|
90
|
+
// protects weak/borderline matches that were never the problem, and does
|
|
91
|
+
// nothing for the strong matches that were.
|
|
92
|
+
//
|
|
93
|
+
// THE ACTUAL FIX: treat dWeight×rFactor the same way RBOOST_CAP treats
|
|
94
|
+
// retrievalBoost — bound it to a small band around 1.0 (a gentle nudge, not
|
|
95
|
+
// an override) via COMPOSITE_DISCOUNT_FLOOR, so durability/recency can never
|
|
96
|
+
// cost a record more than a fixed, small percentage regardless of how stale
|
|
97
|
+
// or low-durability it is. The RBOOST_RELEVANCE_FLOOR-style gate is layered
|
|
98
|
+
// on top per its original intent (no adjustment at all for records that
|
|
99
|
+
// don't even clear a basic relevance bar) but the CAP is what actually stops
|
|
100
|
+
// the magnet: even a `standard`/95-day-old correct answer now loses at most
|
|
101
|
+
// (1 - COMPOSITE_DISCOUNT_FLOOR) of its score, never the 30-95% the
|
|
102
|
+
// unconditional formula could take.
|
|
103
|
+
//
|
|
104
|
+
// COMPOSITE_DISCOUNT_FLOOR (env: FLAIR_COMPOSITE_DISCOUNT_FLOOR) and
|
|
105
|
+
// COMPOSITE_RELEVANCE_FLOOR (env: FLAIR_COMPOSITE_RELEVANCE_FLOOR, default
|
|
106
|
+
// 0.5, same value as RBOOST_RELEVANCE_FLOOR): below the relevance floor the
|
|
107
|
+
// durability/recency multiplier is fully neutral (1.0); at/above it, the
|
|
108
|
+
// multiplier is dWeight×rFactor linearly remapped from its native [0,1] range
|
|
109
|
+
// into [COMPOSITE_DISCOUNT_FLOOR, 1.0] — so a `permanent`+fresh record still
|
|
110
|
+
// scores strictly higher than a `standard`+stale one among relevant
|
|
111
|
+
// candidates (durability/recency remains a real, monotonic signal), but
|
|
112
|
+
// never by more than the bounded amount.
|
|
113
|
+
//
|
|
114
|
+
// DISCOUNT_FLOOR_DEFAULT = 0.98 (max -2%) was tuned empirically against
|
|
115
|
+
// recall-harness's 87-record corpus (test/bench/recall-harness/run.ts,
|
|
116
|
+
// hybrid=true, 3 runs, deterministic/±0.000 SE on this corpus): 0.9 (max
|
|
117
|
+
// -10%, symmetric with RBOOST_CAP's +10%) recovered p@3 to match raw exactly
|
|
118
|
+
// (0.967) but MRR still trailed raw by -0.150 (0.742 vs 0.892) — the bounded
|
|
119
|
+
// discount was still large enough to occasionally reorder within the top-3,
|
|
120
|
+
// just not enough to push the right answer OUT of it. 0.95 narrowed the MRR
|
|
121
|
+
// gap to -0.078. 0.98 closed it to +0.000 on both metrics across all 3 runs
|
|
122
|
+
// — this corpus's RRF-normalized rawScore band is tight enough that even a
|
|
123
|
+
// 5-10% durability/recency swing is bigger than the real relevance gap
|
|
124
|
+
// between candidates. Re-run recall-harness (and recall-eval.mjs on the live
|
|
125
|
+
// corpus) before changing this default or reconsidering scoring defaults.
|
|
126
|
+
export const COMPOSITE_RELEVANCE_FLOOR_DEFAULT = 0.5;
|
|
127
|
+
export const COMPOSITE_DISCOUNT_FLOOR_DEFAULT = 0.98; // max -2% — tuned to fully close the gap to raw on recall-harness
|
|
128
|
+
export function getCompositeRelevanceFloor() {
|
|
129
|
+
const v = Number(process.env.FLAIR_COMPOSITE_RELEVANCE_FLOOR);
|
|
130
|
+
return Number.isFinite(v) && v >= 0 && v <= 1 ? v : COMPOSITE_RELEVANCE_FLOOR_DEFAULT;
|
|
131
|
+
}
|
|
132
|
+
export function getCompositeDiscountFloor() {
|
|
133
|
+
const v = Number(process.env.FLAIR_COMPOSITE_DISCOUNT_FLOOR);
|
|
134
|
+
return Number.isFinite(v) && v >= 0 && v <= 1 ? v : COMPOSITE_DISCOUNT_FLOOR_DEFAULT;
|
|
135
|
+
}
|
|
136
|
+
// flair#683 (2026-07-11, K&S-approved FLAIR-USAGE-FEEDBACK-SIGNAL.md): usage
|
|
137
|
+
// REPLACES retrieval as compositeScore's reinforcement term. Kern's Q1
|
|
138
|
+
// verdict: replace `retrievalBoost` OUTRIGHT rather than weight usage above
|
|
139
|
+
// it — retrievalCount is the CONTAMINATED signal (a search hit counted as
|
|
140
|
+
// "used"); keeping it in the formula at any weight keeps the contamination,
|
|
141
|
+
// because the problem is signal QUALITY, not magnitude. usageCount is
|
|
142
|
+
// strictly more informative (usage implies retrieval happened; retrieval
|
|
143
|
+
// never implies usage). Clean degeneration: on a record with no reported
|
|
144
|
+
// usage yet (the overwhelming majority of the corpus until usage accrues),
|
|
145
|
+
// usageCount=0 → usageBoost=1.0 → compositeScore collapses to
|
|
146
|
+
// semantic × durability/recency, with NO retrieval-popularity pollution at
|
|
147
|
+
// all — a strictly cleaner default state than the pre-#683 formula ever had.
|
|
148
|
+
// retrievalCount/retrievalBoost remain exported above (unused here) —
|
|
149
|
+
// keeping retrievalCount as a fallback "weak prior" for not-yet-used records
|
|
150
|
+
// is an explicitly-deferred v2 idea (FLAIR-USAGE-FEEDBACK-SIGNAL.md's K&S
|
|
151
|
+
// verdict), not built in this slice.
|
|
43
152
|
export function compositeScore(semanticScore, record) {
|
|
44
153
|
const durability = record.durability ?? "standard";
|
|
45
154
|
const dWeight = DURABILITY_WEIGHTS[durability] ?? 0.7;
|
|
46
155
|
const rFactor = record.createdAt ? recencyFactor(record.createdAt, durability) : 1.0;
|
|
47
|
-
//
|
|
48
|
-
//
|
|
49
|
-
//
|
|
50
|
-
|
|
51
|
-
|
|
156
|
+
// Only apply the usage boost when the record is genuinely relevant to this
|
|
157
|
+
// query (semanticScore clears the floor) — same relevance-gate rationale
|
|
158
|
+
// OPS-AYGD established for retrievalBoost: below the floor, a heavily-used
|
|
159
|
+
// doc gets no lift, so a used-but-irrelevant-to-THIS-query record can't
|
|
160
|
+
// magnet its way into an unrelated result set.
|
|
161
|
+
const uBoost = semanticScore >= USAGE_RELEVANCE_FLOOR ? usageBoost(record.usageCount ?? 0) : 1.0;
|
|
162
|
+
// Bound dWeight×rFactor into [discountFloor, 1.0] (a gentle nudge, mirroring
|
|
163
|
+
// RBOOST_CAP), then relevance-gate it (mirroring RBOOST_RELEVANCE_FLOOR): no
|
|
164
|
+
// adjustment at all below the floor, the bounded nudge at/above it.
|
|
165
|
+
const discountFloor = getCompositeDiscountFloor();
|
|
166
|
+
const dWeightRecency = dWeight * rFactor; // native range ~[0, 1]
|
|
167
|
+
const boundedDWeightRecency = discountFloor + (1 - discountFloor) * dWeightRecency; // remapped to [discountFloor, 1]
|
|
168
|
+
const relevanceFloor = getCompositeRelevanceFloor();
|
|
169
|
+
const gatedDWeightRecency = semanticScore >= relevanceFloor ? boundedDWeightRecency : 1.0;
|
|
170
|
+
return semanticScore * gatedDWeightRecency * uBoost;
|
|
52
171
|
}
|