@tpsdev-ai/flair 0.19.0 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +19 -2
- package/dist/deploy.js +188 -8
- package/dist/resources/AdminMemory.js +1 -1
- package/dist/resources/Memory.js +120 -33
- package/dist/resources/MemoryBootstrap.js +99 -22
- package/dist/resources/Presence.js +9 -36
- package/dist/resources/SemanticSearch.js +78 -33
- package/dist/resources/agent-auth.js +10 -34
- package/dist/resources/auth-middleware.js +20 -53
- package/dist/resources/bm25-filter.js +9 -0
- package/dist/resources/dedup.js +24 -0
- package/dist/resources/ed25519-auth.js +119 -0
- package/dist/resources/memory-read-scope.js +112 -0
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -6648,6 +6648,11 @@ program
|
|
|
6648
6648
|
.option("--no-restart", "Do not restart the component after deploy (default: restart=true)")
|
|
6649
6649
|
.option("--dry-run", "Resolve package, validate args, skip the deploy call")
|
|
6650
6650
|
.option("--package-root <dir>", "Override package root (mainly for testing)")
|
|
6651
|
+
.option("--deployment-timeout <ms>", "Milliseconds harper waits for cluster-wide peer replication (env: FABRIC_DEPLOYMENT_TIMEOUT; default: 600000 — harper's own 120s default is too short for Fabric)")
|
|
6652
|
+
.option("--install-timeout <ms>", "Milliseconds harper waits for package install (env: FABRIC_INSTALL_TIMEOUT; default: 600000)")
|
|
6653
|
+
.option("--no-verify", "Skip post-deploy served-API verification (default: verify — on by design, so the CLI can't report success on an empty/broken deploy)")
|
|
6654
|
+
.option("--verify-timeout <ms>", "Milliseconds to wait for the served API to settle after harper's post-deploy restart before verifying (default: 300000)")
|
|
6655
|
+
.option("--verify-resource <name>", "Resource to verify is serving after deploy (repeatable; default: derived from the deployed package's dist/resources)", (val, prev) => [...prev, val], [])
|
|
6651
6656
|
.action(async (opts) => {
|
|
6652
6657
|
const green = (s) => `\x1b[32m${s}\x1b[0m`;
|
|
6653
6658
|
const red = (s) => `\x1b[31m${s}\x1b[0m`;
|
|
@@ -6665,6 +6670,12 @@ program
|
|
|
6665
6670
|
restart: opts.restart !== false,
|
|
6666
6671
|
dryRun: opts.dryRun ?? false,
|
|
6667
6672
|
packageRoot: opts.packageRoot,
|
|
6673
|
+
deploymentTimeoutMs: Number(opts.deploymentTimeout ?? process.env.FABRIC_DEPLOYMENT_TIMEOUT ?? 600_000),
|
|
6674
|
+
installTimeoutMs: Number(opts.installTimeout ?? process.env.FABRIC_INSTALL_TIMEOUT ?? 600_000),
|
|
6675
|
+
verify: opts.verify !== false,
|
|
6676
|
+
verifyResources: opts.verifyResource?.length ? opts.verifyResource : undefined,
|
|
6677
|
+
verifyTimeoutMs: Number(opts.verifyTimeout ?? 300_000),
|
|
6678
|
+
onProgress: (msg) => console.log(dim(` ${msg}`)),
|
|
6668
6679
|
};
|
|
6669
6680
|
const errors = validateDeployOptions(deployOpts);
|
|
6670
6681
|
if (errors.length) {
|
|
@@ -6689,7 +6700,7 @@ program
|
|
|
6689
6700
|
console.log(dim(` package root: ${result.packageRoot}`));
|
|
6690
6701
|
return;
|
|
6691
6702
|
}
|
|
6692
|
-
console.log(`\n${green("✓")} Flair ${result.version} deployed`);
|
|
6703
|
+
console.log(`\n${green("✓")} Flair ${result.version} deployed${deployOpts.verify ? " and verified serving" : ""}`);
|
|
6693
6704
|
console.log(`\n URL: ${result.url}`);
|
|
6694
6705
|
console.log(` Project: ${result.project}`);
|
|
6695
6706
|
console.log(`\nNext steps:`);
|
|
@@ -6703,6 +6714,12 @@ program
|
|
|
6703
6714
|
if (hint?.includes("401") || hint?.includes("unauthoriz")) {
|
|
6704
6715
|
console.error(dim(" hint: check Fabric Studio → Cluster Settings → Admin for the admin password"));
|
|
6705
6716
|
}
|
|
6717
|
+
if (hint?.includes("component is not serving")) {
|
|
6718
|
+
console.error(dim(" hint: harper reported success but the served API disagrees — check the Fabric Studio component logs for the real deploy error, then retry"));
|
|
6719
|
+
}
|
|
6720
|
+
if (hint?.includes("did not settle")) {
|
|
6721
|
+
console.error(dim(" hint: Harper may still be restarting — check Fabric Studio, or retry with a longer --verify-timeout"));
|
|
6722
|
+
}
|
|
6706
6723
|
process.exit(1);
|
|
6707
6724
|
}
|
|
6708
6725
|
});
|
|
@@ -7123,7 +7140,7 @@ memory.command("add [content]").requiredOption("--agent <id>")
|
|
|
7123
7140
|
.option("--summary <text>", "agent-set multi-sentence dense compression (3-tier chain: subject → summary → content; ops-wkoh)")
|
|
7124
7141
|
.option("--subject <text>", "one-line title / entity this memory is about")
|
|
7125
7142
|
.option("--derived-from <csv>", "Comma-separated source Memory IDs this memory was distilled/reflected from (sets Memory.derivedFrom; used by the `rem rapid` reflection loop)")
|
|
7126
|
-
.option("--visibility <value>", "
|
|
7143
|
+
.option("--visibility <value>", "Writer-controlled sharing intent (sets Memory.visibility): 'private' (owner-only, never visible to a grant-holder) or 'shared' (visible to owner + any agent holding a read/search MemoryGrant). Omit to use the server's durability-keyed default: permanent/persistent -> shared, standard/ephemeral -> private (flair#509, ops-2dm3)")
|
|
7127
7144
|
.action(async (contentArg, opts) => {
|
|
7128
7145
|
const content = contentArg ?? opts.content;
|
|
7129
7146
|
if (!content) {
|
package/dist/deploy.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
2
|
import { dirname, join, resolve } from "node:path";
|
|
3
|
-
import { existsSync, readFileSync } from "node:fs";
|
|
3
|
+
import { existsSync, readFileSync, readdirSync } from "node:fs";
|
|
4
4
|
import { fileURLToPath } from "node:url";
|
|
5
5
|
import { createRequire } from "node:module";
|
|
6
6
|
// Files that must be present in a Flair package for deployment.
|
|
@@ -11,6 +11,23 @@ export const REQUIRED_PACKAGE_FILES = [
|
|
|
11
11
|
"ui",
|
|
12
12
|
"config.yaml",
|
|
13
13
|
];
|
|
14
|
+
// harper's own deploy CLI defaults to a 120s peer-replication timeout that's
|
|
15
|
+
// too short for Fabric — the CLI aborts mid-replicate with no override,
|
|
16
|
+
// which is exactly the incident this module now guards against. 10 minutes
|
|
17
|
+
// gives cluster-wide replication + install room to actually finish.
|
|
18
|
+
export const DEFAULT_DEPLOYMENT_TIMEOUT_MS = 600_000;
|
|
19
|
+
export const DEFAULT_INSTALL_TIMEOUT_MS = 600_000;
|
|
20
|
+
// Post-deploy verification: how long we'll wait for the served API to come
|
|
21
|
+
// back up after harper's restart, how often we poll while waiting, and how
|
|
22
|
+
// many consecutive reachable responses count as "settled" (a single
|
|
23
|
+
// reachable probe right after restart can be a fluke mid-flap).
|
|
24
|
+
export const DEFAULT_VERIFY_TIMEOUT_MS = 300_000;
|
|
25
|
+
export const VERIFY_POLL_INTERVAL_MS = 15_000;
|
|
26
|
+
export const VERIFY_SETTLE_STREAK = 3;
|
|
27
|
+
// Fallback when dist/resources can't be scanned (e.g. an unusual package
|
|
28
|
+
// layout via --package-root). Memory is Flair's original, always-present
|
|
29
|
+
// resource — a reasonable single thing to check when derivation fails.
|
|
30
|
+
export const FALLBACK_VERIFY_RESOURCE = "Memory";
|
|
14
31
|
export function validateOptions(opts) {
|
|
15
32
|
const errors = [];
|
|
16
33
|
if (!opts.target) {
|
|
@@ -81,6 +98,65 @@ export function validatePackageLayout(packageRoot) {
|
|
|
81
98
|
missing.join(", "));
|
|
82
99
|
}
|
|
83
100
|
}
|
|
101
|
+
// Derive the list of served, table-backed REST resources from the compiled
|
|
102
|
+
// package — no hardcoded resource list. Flair's jsResource files (dist/resources/*.js)
|
|
103
|
+
// contain both real Resource classes (routable, GET-able) and plain helper
|
|
104
|
+
// modules (embeddings, auth, scoring, etc). We only ship dist/ in the
|
|
105
|
+
// published package (resources/*.ts source is not in package.json's `files`),
|
|
106
|
+
// so this scans the COMPILED output, matching the convention every current
|
|
107
|
+
// table-backed resource follows:
|
|
108
|
+
//
|
|
109
|
+
// export class <Name> extends databases.<db>.<Name> { ... }
|
|
110
|
+
//
|
|
111
|
+
// i.e. the exported class name equals the filename equals the underlying
|
|
112
|
+
// table name (Memory.js -> `export class Memory extends databases.flair.Memory`,
|
|
113
|
+
// same for Agent, Soul, MemoryGrant, Credential, OrgEvent, etc). Helper
|
|
114
|
+
// modules are lowercase-first (agent-auth.js, embeddings-provider.js, ...)
|
|
115
|
+
// and never match, so they're skipped without needing a denylist. Files
|
|
116
|
+
// that export a resource extending a *generic* `Resource` base (AgentCard,
|
|
117
|
+
// WorkspaceLatest, action-style endpoints) are deliberately excluded here —
|
|
118
|
+
// they're action/command endpoints, not GET-able collections, and asserting
|
|
119
|
+
// non-404 on them would be the wrong check.
|
|
120
|
+
// Literal regex (no interpolation — satisfies semgrep detect-non-literal-regexp):
|
|
121
|
+
// matches `export class <Name> extends databases.<db>.<Name>` where the `\1`
|
|
122
|
+
// backreference forces the class name and the table name to be identical.
|
|
123
|
+
// Capture group 1 is the resource name; the caller matches it against the filename.
|
|
124
|
+
const EXPORTED_TABLE_CLASS_RE = /export class (\w+) extends databases\.[A-Za-z_$][\w$]*\.\1\b/g;
|
|
125
|
+
export function deriveVerifyResources(packageRoot) {
|
|
126
|
+
const resourcesDir = join(packageRoot, "dist", "resources");
|
|
127
|
+
let entries;
|
|
128
|
+
try {
|
|
129
|
+
entries = readdirSync(resourcesDir);
|
|
130
|
+
}
|
|
131
|
+
catch {
|
|
132
|
+
return [FALLBACK_VERIFY_RESOURCE];
|
|
133
|
+
}
|
|
134
|
+
const names = [];
|
|
135
|
+
for (const entry of entries) {
|
|
136
|
+
if (!entry.endsWith(".js"))
|
|
137
|
+
continue;
|
|
138
|
+
const base = entry.slice(0, -3);
|
|
139
|
+
if (!/^[A-Z]/.test(base))
|
|
140
|
+
continue; // helper modules are camelCase/lowercase
|
|
141
|
+
let src;
|
|
142
|
+
try {
|
|
143
|
+
src = readFileSync(join(resourcesDir, entry), "utf8");
|
|
144
|
+
}
|
|
145
|
+
catch {
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
// The file serves a table resource iff it exports a class whose name matches
|
|
149
|
+
// its filename (base) and extends databases.<db>.<sameName>.
|
|
150
|
+
for (const m of src.matchAll(EXPORTED_TABLE_CLASS_RE)) {
|
|
151
|
+
if (m[1] === base) {
|
|
152
|
+
names.push(base);
|
|
153
|
+
break;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
names.sort();
|
|
158
|
+
return names.length ? names : [FALLBACK_VERIFY_RESOURCE];
|
|
159
|
+
}
|
|
84
160
|
function resolveHarperBin(packageRoot) {
|
|
85
161
|
const local = join(packageRoot, "node_modules/@harperfast/harper/dist/bin/harper.js");
|
|
86
162
|
if (existsSync(local))
|
|
@@ -105,6 +181,22 @@ function resolveHarperBin(packageRoot) {
|
|
|
105
181
|
throw new Error("Could not locate Harper CLI binary (@harperfast/harper). " +
|
|
106
182
|
"Flair deploy requires Harper to be installed alongside Flair.");
|
|
107
183
|
}
|
|
184
|
+
// Pure arg-array builder — separated from spawnHarper so the timeout
|
|
185
|
+
// passthrough (and the rest of the arg shape) is unit-testable without
|
|
186
|
+
// mocking child_process / actually spawning harper.
|
|
187
|
+
export function buildHarperDeployArgs(opts, url, project) {
|
|
188
|
+
const deploymentTimeoutMs = opts.deploymentTimeoutMs ?? DEFAULT_DEPLOYMENT_TIMEOUT_MS;
|
|
189
|
+
const installTimeoutMs = opts.installTimeoutMs ?? DEFAULT_INSTALL_TIMEOUT_MS;
|
|
190
|
+
return [
|
|
191
|
+
"deploy",
|
|
192
|
+
`target=${url}`,
|
|
193
|
+
`project=${project}`,
|
|
194
|
+
`restart=${opts.restart !== false}`,
|
|
195
|
+
`replicated=${opts.replicated !== false}`,
|
|
196
|
+
`deployment_timeout=${deploymentTimeoutMs}`,
|
|
197
|
+
`install_timeout=${installTimeoutMs}`,
|
|
198
|
+
];
|
|
199
|
+
}
|
|
108
200
|
function spawnHarper(bin, args, cwd, env) {
|
|
109
201
|
return new Promise((resolveP, rejectP) => {
|
|
110
202
|
const p = spawn(process.execPath, [bin, ...args], {
|
|
@@ -121,6 +213,85 @@ function spawnHarper(bin, args, cwd, env) {
|
|
|
121
213
|
});
|
|
122
214
|
});
|
|
123
215
|
}
|
|
216
|
+
function sleep(ms) {
|
|
217
|
+
return new Promise((r) => setTimeout(r, ms));
|
|
218
|
+
}
|
|
219
|
+
// A single reachability probe against the served (REST) base URL. Harper
|
|
220
|
+
// restarts the process after every deploy, so the endpoint FLAPS for a bit —
|
|
221
|
+
// connection refused / reset / DNS blips are all EXPECTED right after
|
|
222
|
+
// restart, not a failure signal by themselves. Any HTTP response at all
|
|
223
|
+
// (regardless of status code — a 404 on `/` is normal) proves the process
|
|
224
|
+
// is back up and terminating TLS/HTTP again; that's all "reachable" means
|
|
225
|
+
// here. Resource-level 404s are a separate, later check.
|
|
226
|
+
async function probeReachable(baseUrl, fetchImpl) {
|
|
227
|
+
try {
|
|
228
|
+
await fetchImpl(baseUrl, { method: "GET", signal: AbortSignal.timeout(10_000) });
|
|
229
|
+
return true;
|
|
230
|
+
}
|
|
231
|
+
catch {
|
|
232
|
+
return false;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
async function pollUntilSettled(baseUrl, timeoutMs, pollIntervalMs, settleStreak, fetchImpl, onProgress) {
|
|
236
|
+
const deadline = Date.now() + timeoutMs;
|
|
237
|
+
let streak = 0;
|
|
238
|
+
let attempt = 0;
|
|
239
|
+
for (;;) {
|
|
240
|
+
attempt++;
|
|
241
|
+
const reachable = await probeReachable(baseUrl, fetchImpl);
|
|
242
|
+
streak = reachable ? streak + 1 : 0;
|
|
243
|
+
if (!reachable) {
|
|
244
|
+
onProgress?.(`waiting for ${baseUrl} to come back up after restart (attempt ${attempt})...`);
|
|
245
|
+
}
|
|
246
|
+
if (streak >= settleStreak)
|
|
247
|
+
return;
|
|
248
|
+
if (Date.now() >= deadline) {
|
|
249
|
+
throw new Error(`deploy verification: ${baseUrl} did not settle within ${timeoutMs}ms after restart ` +
|
|
250
|
+
`(Harper never came back up, or is unusually slow to restart post-deploy)`);
|
|
251
|
+
}
|
|
252
|
+
await sleep(pollIntervalMs);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
async function verifyResourcesServing(baseUrl, resources, fetchImpl) {
|
|
256
|
+
const base = baseUrl.replace(/\/+$/, "");
|
|
257
|
+
const notServing = [];
|
|
258
|
+
for (const name of resources) {
|
|
259
|
+
const path = `${base}/${name}`;
|
|
260
|
+
let status;
|
|
261
|
+
try {
|
|
262
|
+
const res = await fetchImpl(path, { method: "GET", signal: AbortSignal.timeout(10_000) });
|
|
263
|
+
status = res.status;
|
|
264
|
+
}
|
|
265
|
+
catch (err) {
|
|
266
|
+
throw new Error(`deploy verification: request to ${path} failed even after the endpoint settled: ${err?.message ?? err}`);
|
|
267
|
+
}
|
|
268
|
+
// 404 = the resource genuinely isn't being served (this is the incident:
|
|
269
|
+
// harper reported "Successfully deployed" while the component was empty).
|
|
270
|
+
// 401 = auth-gated, which means the resource IS being served correctly.
|
|
271
|
+
// 200 = served + accessible. Both count as pass.
|
|
272
|
+
if (status === 404)
|
|
273
|
+
notServing.push(name);
|
|
274
|
+
}
|
|
275
|
+
if (notServing.length) {
|
|
276
|
+
const list = notServing.map((n) => `/${n}`).join(", ");
|
|
277
|
+
throw new Error(`deploy reported success but ${list} return${notServing.length === 1 ? "s" : ""} 404 — ` +
|
|
278
|
+
`component is not serving; likely deployed the wrong package root`);
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
// The tool must not be able to lie. harper's deploy CLI can print
|
|
282
|
+
// "Successfully deployed" for an empty component — the only way to know the
|
|
283
|
+
// deploy actually worked is to curl the served API and check it isn't 404.
|
|
284
|
+
// This polls the served base URL (443, NOT the :9925 ops API deploy talks
|
|
285
|
+
// to) until it settles after harper's post-deploy restart, then asserts the
|
|
286
|
+
// derived resource(s) respond non-404.
|
|
287
|
+
export async function verifyDeployServing(o) {
|
|
288
|
+
const { baseUrl, resources, timeoutMs = DEFAULT_VERIFY_TIMEOUT_MS, pollIntervalMs = VERIFY_POLL_INTERVAL_MS, settleStreak = VERIFY_SETTLE_STREAK, fetchImpl = fetch, onProgress, } = o;
|
|
289
|
+
onProgress?.(`verifying ${baseUrl} is actually serving (not just reported deployed)...`);
|
|
290
|
+
await pollUntilSettled(baseUrl, timeoutMs, pollIntervalMs, settleStreak, fetchImpl, onProgress);
|
|
291
|
+
onProgress?.(`settled — checking ${resources.map((r) => `/${r}`).join(", ")}...`);
|
|
292
|
+
await verifyResourcesServing(baseUrl, resources, fetchImpl);
|
|
293
|
+
onProgress?.(`served API verified non-404 for ${resources.length} resource(s)`);
|
|
294
|
+
}
|
|
124
295
|
export async function deploy(opts) {
|
|
125
296
|
const errors = validateOptions(opts);
|
|
126
297
|
if (errors.length) {
|
|
@@ -141,13 +312,7 @@ export async function deploy(opts) {
|
|
|
141
312
|
"Pass --fabric-user + --fabric-password instead.");
|
|
142
313
|
}
|
|
143
314
|
const harperBin = resolveHarperBin(packageRoot);
|
|
144
|
-
const args =
|
|
145
|
-
"deploy",
|
|
146
|
-
`target=${url}`,
|
|
147
|
-
`project=${project}`,
|
|
148
|
-
`restart=${opts.restart !== false}`,
|
|
149
|
-
`replicated=${opts.replicated !== false}`,
|
|
150
|
-
];
|
|
315
|
+
const args = buildHarperDeployArgs(opts, url, project);
|
|
151
316
|
// Credentials go via env, not argv, so they don't appear in `ps` output
|
|
152
317
|
// for the lifetime of the Harper child process. Harper's cliOperations
|
|
153
318
|
// reads CLI_TARGET_USERNAME / CLI_TARGET_PASSWORD as env fallbacks.
|
|
@@ -157,5 +322,20 @@ export async function deploy(opts) {
|
|
|
157
322
|
CLI_TARGET_PASSWORD: opts.fabricPassword,
|
|
158
323
|
};
|
|
159
324
|
await spawnHarper(harperBin, args, packageRoot, childEnv);
|
|
325
|
+
// harper can print "Successfully deployed" for a component that isn't
|
|
326
|
+
// actually serving anything (the incident this closes: an empty deploy,
|
|
327
|
+
// reported success, /Memory 404ing in prod). Verify by curling the served
|
|
328
|
+
// API — on by default, escape hatch via --no-verify.
|
|
329
|
+
if (opts.verify !== false) {
|
|
330
|
+
const resources = opts.verifyResources?.length
|
|
331
|
+
? opts.verifyResources
|
|
332
|
+
: deriveVerifyResources(packageRoot);
|
|
333
|
+
await verifyDeployServing({
|
|
334
|
+
baseUrl: url,
|
|
335
|
+
resources,
|
|
336
|
+
timeoutMs: opts.verifyTimeoutMs ?? DEFAULT_VERIFY_TIMEOUT_MS,
|
|
337
|
+
onProgress: opts.onProgress,
|
|
338
|
+
});
|
|
339
|
+
}
|
|
160
340
|
return { url, project, version, packageRoot, dryRun: false };
|
|
161
341
|
}
|
|
@@ -342,7 +342,7 @@ export class AdminMemory extends Resource {
|
|
|
342
342
|
<dl style="display:grid;grid-template-columns:200px 1fr;gap:6px;margin-top:8px;font-family:ui-monospace,SF Mono,monospace;font-size:0.85em">
|
|
343
343
|
<dt style="color:#666">id</dt><dd>${esc(m.id)}</dd>
|
|
344
344
|
<dt style="color:#666">contentHash</dt><dd>${esc(m.contentHash ?? "—")}</dd>
|
|
345
|
-
<dt style="color:#666">visibility</dt><dd>${esc(m.visibility ?? "
|
|
345
|
+
<dt style="color:#666">visibility</dt><dd>${esc(m.visibility ?? "shared (no field — pre-migration default)")}</dd>
|
|
346
346
|
</dl>
|
|
347
347
|
</div>
|
|
348
348
|
`;
|
package/dist/resources/Memory.js
CHANGED
|
@@ -4,31 +4,23 @@ import { isAdmin, resolveAgentAuth, allowVerified } from "./agent-auth.js";
|
|
|
4
4
|
import { getEmbedding, getModelId } from "./embeddings-provider.js";
|
|
5
5
|
import { scanFields, isStrictMode } from "./content-safety.js";
|
|
6
6
|
import { checkRateLimit, rateLimitResponse } from "./rate-limiter.js";
|
|
7
|
-
import {
|
|
7
|
+
import { resolveReadScope } from "./memory-read-scope.js";
|
|
8
|
+
import { DEDUP_COSINE_THRESHOLD_DEFAULT, DEDUP_LEXICAL_THRESHOLD_DEFAULT, DEDUP_MIN_CONTENT_LENGTH, computeMatchConfidence, cosineSimilarity, isConservativeMatch, } from "./dedup.js";
|
|
8
9
|
const FORBIDDEN = (msg) => new Response(JSON.stringify({ error: msg }), { status: 403, headers: { "Content-Type": "application/json" } });
|
|
9
10
|
const UNAUTH = () => new Response(JSON.stringify({ error: "authentication required" }), { status: 401, headers: { "Content-Type": "application/json" } });
|
|
10
11
|
const NOT_FOUND = () => new Response(JSON.stringify({ error: "not found" }), { status: 404, headers: { "Content-Type": "application/json" } });
|
|
11
12
|
/**
|
|
12
|
-
* Owner ids a non-admin agent may READ
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
13
|
+
* Owner ids a non-admin agent may READ (resolveAllowedOwners) and the full
|
|
14
|
+
* read-scope condition + private-exclusion predicate (resolveReadScope) now
|
|
15
|
+
* live in ./memory-read-scope.ts — the ONE centralized helper every
|
|
16
|
+
* cross-agent Memory read path (search()/get() here, SemanticSearch.ts,
|
|
17
|
+
* MemoryBootstrap.ts, auth-middleware.ts's by-id guard) resolves its scope
|
|
18
|
+
* through, so the scoping rule cannot drift per-path again (ops-nzxa: a
|
|
19
|
+
* SemanticSearch inline `visibility === "office"` OR-clause leaked office
|
|
20
|
+
* memories to any authenticated agent because the rule had scattered). See
|
|
21
|
+
* that module's doc for the migration invariant (no-visibility-field reads
|
|
22
|
+
* as "shared", never "private").
|
|
17
23
|
*/
|
|
18
|
-
async function resolveAllowedOwners(authAgentId) {
|
|
19
|
-
const allowedOwners = [authAgentId];
|
|
20
|
-
try {
|
|
21
|
-
for await (const grant of databases.flair.MemoryGrant.search({
|
|
22
|
-
conditions: [{ attribute: "granteeId", comparator: "equals", value: authAgentId }],
|
|
23
|
-
})) {
|
|
24
|
-
if (grant.ownerId && (grant.scope === "read" || grant.scope === "search")) {
|
|
25
|
-
allowedOwners.push(grant.ownerId);
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
catch { /* MemoryGrant table not yet populated — ignore */ }
|
|
30
|
-
return allowedOwners;
|
|
31
|
-
}
|
|
32
24
|
/**
|
|
33
25
|
* ─── Server-side conservative-duplicate gate (memory-integrity fix) ──────────
|
|
34
26
|
*
|
|
@@ -94,7 +86,59 @@ async function findConservativeDedupMatch(ctx, agentId, contentText, embedding,
|
|
|
94
86
|
}
|
|
95
87
|
if (!top)
|
|
96
88
|
return null;
|
|
97
|
-
|
|
89
|
+
// ─── ops-ume4: Harper's cosine-sort query omits $distance for a SINGLETON
|
|
90
|
+
// result set ─────────────────────────────────────────────────────────────
|
|
91
|
+
// Initial working theory was a per-agentId HNSW "cold-start" (first-ever
|
|
92
|
+
// query cold, second query warm) and the initially-recommended fix was a
|
|
93
|
+
// same-query retry. Empirically FALSIFIED: a plain retry of the identical
|
|
94
|
+
// query, 8x with 300ms delays (2.4s total), never recovered a `$distance`
|
|
95
|
+
// for a genuinely singleton candidate set (exactly one record matching
|
|
96
|
+
// `agentId equals X AND archived not_equal true`). The actual trigger,
|
|
97
|
+
// confirmed by direct probing: when this query's post-filter result set
|
|
98
|
+
// has exactly ONE matching record, `$distance` comes back `undefined` for
|
|
99
|
+
// it — regardless of how many prior queries have run for that agentId,
|
|
100
|
+
// how long you wait, or how many other agentIds/records already exist in
|
|
101
|
+
// the table. The moment a SECOND matching record exists, `$distance` is
|
|
102
|
+
// populated correctly on the very first query ever issued for that
|
|
103
|
+
// agentId — no warm-up needed. In practice the singleton case is exactly
|
|
104
|
+
// an agent's SECOND-ever memory (compared against their first) — the most
|
|
105
|
+
// common real-world trigger for this bug, and why it looked "permanent
|
|
106
|
+
// per-agent for the first near-dup query."
|
|
107
|
+
//
|
|
108
|
+
// Also NOT a query-shape/conditions issue: the `{operator:"or"}` wrap
|
|
109
|
+
// SemanticSearch.ts uses elsewhere is unrelated, and neither raising
|
|
110
|
+
// `limit` past 1 nor changing the conditions shape changes the result —
|
|
111
|
+
// confirmed empirically. Harper's SORT ordering is correct even in the
|
|
112
|
+
// singleton case (the right record comes back as `top`); only the
|
|
113
|
+
// numeric `$distance` annotation is missing. Also confirmed: selecting
|
|
114
|
+
// "embedding" directly on THIS sort-by-embedding query does not help
|
|
115
|
+
// either — it comes back as a bare scalar (Harper appears to special-case
|
|
116
|
+
// the sort attribute in `select`), not the stored vector.
|
|
117
|
+
//
|
|
118
|
+
// Fix: when `$distance` is undefined, fetch the ONE candidate's full
|
|
119
|
+
// record by id (a plain point lookup — not a vector-sort query, so
|
|
120
|
+
// unaffected by the quirk above) and compute cosine similarity ourselves
|
|
121
|
+
// in JS from its real stored `embedding` vector against this write's own
|
|
122
|
+
// `embedding` (this function's parameter), via the same math Harper would
|
|
123
|
+
// have used (dedup.ts's cosineSimilarity, Harper-free and unit-tested).
|
|
124
|
+
// This sidesteps the underlying engine quirk entirely rather than
|
|
125
|
+
// depending on its timing, and works identically whether this is the
|
|
126
|
+
// agent's first query ever or its thousandth. Never suppresses the write
|
|
127
|
+
// either way: if the candidate's embedding is somehow also missing (e.g.
|
|
128
|
+
// a legacy record written before embeddings existed), `cosineSimilarity`
|
|
129
|
+
// returns 0 — the same safe "no match" signal the pre-fix `?? 1`
|
|
130
|
+
// fallback produced.
|
|
131
|
+
let cosine;
|
|
132
|
+
if (top.$distance !== undefined) {
|
|
133
|
+
cosine = 1 - top.$distance;
|
|
134
|
+
}
|
|
135
|
+
else {
|
|
136
|
+
console.error("Memory.findConservativeDedupMatch: $distance undefined on a singleton cosine result (ops-ume4) — " +
|
|
137
|
+
"falling back to a manual cosine computation from the candidate's stored embedding", { agentId, candidateId: top.id });
|
|
138
|
+
const fullCandidate = await withDetachedTxn(ctx, () => databases.flair.Memory.get(top.id));
|
|
139
|
+
const candidateEmbedding = Array.isArray(fullCandidate?.embedding) ? fullCandidate.embedding : [];
|
|
140
|
+
cosine = cosineSimilarity(embedding, candidateEmbedding);
|
|
141
|
+
}
|
|
98
142
|
const confidence = computeMatchConfidence(contentText, top.content, cosine);
|
|
99
143
|
if (!isConservativeMatch(confidence.cosine, confidence.lexical, cosineThreshold, lexicalThreshold)) {
|
|
100
144
|
return null;
|
|
@@ -252,6 +296,20 @@ async function closeSupersededIfNeeded(ctx, content, methodLabel) {
|
|
|
252
296
|
"(ops-a4t5 — observable, not silent; new record is safely written, old record remains active until retried)", { method: methodLabel, supersededId: content.supersedes, newRecordId: content.id, err });
|
|
253
297
|
}
|
|
254
298
|
}
|
|
299
|
+
/**
|
|
300
|
+
* ─── Durability-keyed default visibility (ops-2dm3 Layer 1, part A) ─────────
|
|
301
|
+
*
|
|
302
|
+
* Writer intent: an explicit `visibility` on the write ALWAYS overrides this
|
|
303
|
+
* (callers check `content.visibility == null` before calling this). When
|
|
304
|
+
* unset, the default is keyed off durability — a durable write (the agent
|
|
305
|
+
* chose to make this stick around) defaults to shared; anything else
|
|
306
|
+
* defaults to private. "absent" durability (not yet defaulted by the caller)
|
|
307
|
+
* falls into the private branch, matching the spec's
|
|
308
|
+
* "standard|ephemeral|absent → private".
|
|
309
|
+
*/
|
|
310
|
+
function defaultVisibilityForDurability(durability) {
|
|
311
|
+
return durability === "permanent" || durability === "persistent" ? "shared" : "private";
|
|
312
|
+
}
|
|
255
313
|
export class Memory extends databases.flair.Memory {
|
|
256
314
|
/**
|
|
257
315
|
* Self-authorize now that the global gate is non-rejecting. Closes the P0
|
|
@@ -301,14 +359,15 @@ export class Memory extends databases.flair.Memory {
|
|
|
301
359
|
if (auth.kind === "internal" || (auth.kind === "agent" && auth.isAdmin)) {
|
|
302
360
|
return super.get(target);
|
|
303
361
|
}
|
|
304
|
-
// Non-admin agent: only its own memories, or
|
|
305
|
-
//
|
|
306
|
-
//
|
|
362
|
+
// Non-admin agent: only its own memories (any visibility), or a granted
|
|
363
|
+
// owner's SHARED memories — never that owner's private ones (ops-2dm3
|
|
364
|
+
// Layer 1 private-exclusion). Centralized in resolveReadScope() so this
|
|
365
|
+
// and search() below cannot drift.
|
|
307
366
|
const record = await super.get(target);
|
|
308
367
|
if (!record)
|
|
309
368
|
return NOT_FOUND();
|
|
310
|
-
const
|
|
311
|
-
if (!
|
|
369
|
+
const scope = await resolveReadScope(auth.agentId);
|
|
370
|
+
if (!scope.isAllowed(record))
|
|
312
371
|
return NOT_FOUND();
|
|
313
372
|
return record;
|
|
314
373
|
}
|
|
@@ -337,13 +396,12 @@ export class Memory extends databases.flair.Memory {
|
|
|
337
396
|
if (auth.kind === "internal" || (auth.kind === "agent" && auth.isAdmin)) {
|
|
338
397
|
return super.search(query);
|
|
339
398
|
}
|
|
340
|
-
// Non-admin agent: scope to own + granted owners
|
|
399
|
+
// Non-admin agent: scope to own (any visibility) + granted owners' SHARED
|
|
400
|
+
// memories only (ops-2dm3 Layer 1 private-exclusion). Centralized in
|
|
401
|
+
// resolveReadScope() so get() above and search() here cannot drift.
|
|
341
402
|
const authAgent = auth.agentId;
|
|
342
|
-
const
|
|
343
|
-
|
|
344
|
-
const agentIdCondition = allowedOwners.length === 1
|
|
345
|
-
? { attribute: "agentId", comparator: "equals", value: allowedOwners[0] }
|
|
346
|
-
: { conditions: allowedOwners.map(id => ({ attribute: "agentId", comparator: "equals", value: id })), operator: "or" };
|
|
403
|
+
const scope = await resolveReadScope(authAgent);
|
|
404
|
+
const agentIdCondition = scope.condition;
|
|
347
405
|
// Harper passes `query` as a RequestTarget (extends URLSearchParams) or a
|
|
348
406
|
// conditions array. For URL-based GET /Memory?... calls, URL params are no
|
|
349
407
|
// longer translated to conditions here — callers should use
|
|
@@ -392,6 +450,16 @@ export class Memory extends databases.flair.Memory {
|
|
|
392
450
|
content.createdAt = new Date().toISOString();
|
|
393
451
|
content.updatedAt = content.createdAt;
|
|
394
452
|
content.archived = content.archived ?? false;
|
|
453
|
+
// ─── Default visibility (durability-keyed) — ops-2dm3 Layer 1, part A ────
|
|
454
|
+
// post() only ever creates a NEW record — patchRecord/supersede-close/
|
|
455
|
+
// retrievalCount bumps all route through put() instead (see put()'s
|
|
456
|
+
// pre-existing-record guard below), so there is no "don't overwrite an
|
|
457
|
+
// existing record's visibility" concern here. Explicit visibility on the
|
|
458
|
+
// write ALWAYS overrides; only stamp the default when the caller left it
|
|
459
|
+
// unset. permanent|persistent → shared; standard|ephemeral|absent → private.
|
|
460
|
+
if (content.visibility === undefined || content.visibility === null) {
|
|
461
|
+
content.visibility = defaultVisibilityForDurability(content.durability);
|
|
462
|
+
}
|
|
395
463
|
// Validate derivedFrom source IDs exist (best-effort, non-blocking)
|
|
396
464
|
if (Array.isArray(content.derivedFrom) && content.derivedFrom.length > 0) {
|
|
397
465
|
const now = content.createdAt;
|
|
@@ -509,6 +577,26 @@ export class Memory extends databases.flair.Memory {
|
|
|
509
577
|
// Set defaults that post() sets — put() is also used for new records via CLI
|
|
510
578
|
content.archived = content.archived ?? false;
|
|
511
579
|
content.createdAt = content.createdAt ?? now;
|
|
580
|
+
// Fetch the pre-existing record (if any) ONCE — reused below both to
|
|
581
|
+
// decide whether this PUT is a fresh create (dedup gate applies, default
|
|
582
|
+
// visibility stamped) or an update/patch (dedup-bypassed, visibility left
|
|
583
|
+
// untouched). See the dedup-gate block further down for why an existing
|
|
584
|
+
// id skips the gate; the SAME "does a record already exist" check gates
|
|
585
|
+
// the visibility default (ops-2dm3 Layer 1 part A): patchRecord/supersede-
|
|
586
|
+
// close/retrievalCount bumps all route through put() with a MERGED
|
|
587
|
+
// `{...existing, ...patch}` payload, and must never have their stored
|
|
588
|
+
// visibility overwritten by a default recomputed from that merged content
|
|
589
|
+
// — only a genuinely NEW id gets the default stamped.
|
|
590
|
+
const preExisting = content.id
|
|
591
|
+
? await databases.flair.Memory.get(content.id).catch(() => null)
|
|
592
|
+
: null;
|
|
593
|
+
// ─── Default visibility (durability-keyed) — ops-2dm3 Layer 1, part A ────
|
|
594
|
+
// Explicit visibility on the write ALWAYS overrides; only stamp the
|
|
595
|
+
// default when the caller left it unset AND this is a fresh record.
|
|
596
|
+
// permanent|persistent → shared; standard|ephemeral|absent → private.
|
|
597
|
+
if (!preExisting && (content.visibility === undefined || content.visibility === null)) {
|
|
598
|
+
content.visibility = defaultVisibilityForDurability(content.durability);
|
|
599
|
+
}
|
|
512
600
|
// supersedes: optional reference to the ID of the memory this one
|
|
513
601
|
// replaces. Validates shape + cross-agent-write authorization (shared
|
|
514
602
|
// with post() — see validateAndAuthorizeSupersedes doc for why PUT needs
|
|
@@ -552,7 +640,6 @@ export class Memory extends databases.flair.Memory {
|
|
|
552
640
|
delete content.lexicalThreshold;
|
|
553
641
|
}
|
|
554
642
|
else if (content.id) {
|
|
555
|
-
const preExisting = await databases.flair.Memory.get(content.id).catch(() => null);
|
|
556
643
|
if (!preExisting) {
|
|
557
644
|
dedupMatch = await runDedupGate(ctx, content);
|
|
558
645
|
}
|