@tpsdev-ai/flair 0.44.9 → 0.44.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +188 -2
- package/dist/rem/runner.js +167 -23
- package/dist/resources/MemoryBootstrap.js +197 -80
- package/dist/resources/MemoryReflect.js +15 -13
- package/dist/resources/mcp-tools.js +242 -19
- package/dist/resources/memory-bootstrap-lib.js +58 -0
- package/dist/resources/memory-reflect-lib.js +70 -0
- package/dist/resources/token-estimate.js +25 -0
- package/docs/rem.md +6 -0
- package/package.json +1 -1
- package/schemas/memory.graphql +9 -0
package/dist/cli.js
CHANGED
|
@@ -7455,6 +7455,133 @@ export function decideCandidateAction(candidate, action) {
|
|
|
7455
7455
|
}
|
|
7456
7456
|
return { ok: true };
|
|
7457
7457
|
}
|
|
7458
|
+
// ─── ADK tag-lineage on promote (#1205 slice 1205a — Sherlock security req) ───
|
|
7459
|
+
// ADK session records are written by adk-flair (memory_service.py) under a
|
|
7460
|
+
// SHARED-namespace agentId, with per-user separation carried ENTIRELY by a
|
|
7461
|
+
// compound scope tag `adk:<app>:<user>`. That tag is the access-control
|
|
7462
|
+
// boundary. A candidate distilled from those records therefore MUST carry the
|
|
7463
|
+
// scope tag when promoted, or the promoted claim lands in the shared agentId
|
|
7464
|
+
// memory retrievable by every other user of the app — a cross-user leak.
|
|
7465
|
+
//
|
|
7466
|
+
// `rem promote` historically hard-coded `["nightly-rem-promoted", from:<id>]`
|
|
7467
|
+
// and DROPPED the source tag. We now propagate the source scope tag for
|
|
7468
|
+
// ADK-sourced candidates, and FAIL CLOSED (refuse) when a candidate is
|
|
7469
|
+
// ADK-sourced but its scope tag can't be uniquely+completely determined.
|
|
7470
|
+
//
|
|
7471
|
+
// SCOPING (deliberate, per spec): fail-closed applies ONLY to ADK-sourced
|
|
7472
|
+
// candidates. Non-ADK candidates carry no `adk:` tag and promote byte-for-byte
|
|
7473
|
+
// as before — a transient/deleted source on a non-ADK candidate must NOT block
|
|
7474
|
+
// its promotion.
|
|
7475
|
+
//
|
|
7476
|
+
// SEAM (foundation only; the distillation engine is slice #1205b): ADK-sourcing
|
|
7477
|
+
// is detected here by re-reading the candidate's source memories and inspecting
|
|
7478
|
+
// their tags. That leaves ONE residual fail-open: an ADK-sourced candidate all
|
|
7479
|
+
// of whose source memories are unreadable (deleted/transient) yields no `adk:`
|
|
7480
|
+
// evidence and is treated as non-ADK. Closing that corner without regressing
|
|
7481
|
+
// non-ADK promotion requires the ENGINE to stamp the authoritative scope tag
|
|
7482
|
+
// onto the MemoryCandidate row at distillation time (it distills per single
|
|
7483
|
+
// scope:tagged tag, so it knows it authoritatively). `derivePromotedTags` is
|
|
7484
|
+
// written so that override can be threaded in later without touching callers.
|
|
7485
|
+
export const ADK_SCOPE_TAG_PREFIX = "adk:";
|
|
7486
|
+
/**
|
|
7487
|
+
* Decide the tag set for a promoted Memory given the candidate id and the
|
|
7488
|
+
* result of fetching each of its source memories. Pure — no I/O; the action
|
|
7489
|
+
* callback does the fetching and threads the results here so this is unit-
|
|
7490
|
+
* testable and the fail-closed logic is exercised directly.
|
|
7491
|
+
*
|
|
7492
|
+
* `stampedScopeTag` (#1205b-1 — the engine slice the #1205a SEAM note below
|
|
7493
|
+
* anticipated): the authoritative scope:"tagged" tag the distillation engine
|
|
7494
|
+
* stamped onto the MemoryCandidate row (resources/MemoryReflect.ts →
|
|
7495
|
+
* buildStagedCandidateRow). When present it is AUTHORITATIVE and short-circuits
|
|
7496
|
+
* the source re-read entirely — the engine distilled under exactly this one
|
|
7497
|
+
* tag, so it knows the per-user scope tag independent of whether the source
|
|
7498
|
+
* memories are still readable. This closes the residual fail-open the SEAM
|
|
7499
|
+
* note describes: a candidate all of whose sources are unreadable yields no
|
|
7500
|
+
* `adk:` evidence and would otherwise be mis-classified NON-ADK and promoted
|
|
7501
|
+
* tagless into the shared agentId namespace (a cross-user leak). Threading it
|
|
7502
|
+
* in as an optional trailing arg keeps every pre-#1205b caller (and every
|
|
7503
|
+
* candidate that never carried a stamp) on the unchanged source-re-read path.
|
|
7504
|
+
*
|
|
7505
|
+
* With NO stamp (undefined/empty) the source-re-read classification runs
|
|
7506
|
+
* exactly as in #1205a:
|
|
7507
|
+
* - No `adk:` scope tag across readable sources → NON-ADK candidate; return
|
|
7508
|
+
* the provenance tags only (unchanged behavior).
|
|
7509
|
+
* - Exactly one `adk:` scope tag AND every source readable → ADK-sourced;
|
|
7510
|
+
* return [scopeTag, ...provenance].
|
|
7511
|
+
* - `adk:` evidence present but the scope tag is ambiguous (>1 distinct tag)
|
|
7512
|
+
* OR incomplete (some source unreadable) → REFUSE (fail-closed): a
|
|
7513
|
+
* tagless/mis-tagged claim in a shared ADK namespace is a cross-user leak,
|
|
7514
|
+
* not a benign miss.
|
|
7515
|
+
*/
|
|
7516
|
+
export function derivePromotedTags(candidateId, sources, stampedScopeTag) {
|
|
7517
|
+
const provenance = ["nightly-rem-promoted", `from:${candidateId}`];
|
|
7518
|
+
// #1205b-1: a stamped scope tag is AUTHORITATIVE — consume it directly, never
|
|
7519
|
+
// re-read sources. This is the seam closure: correctness no longer depends on
|
|
7520
|
+
// source readability. `adkSourced` (which gates the Soul-promotion refusal in
|
|
7521
|
+
// the promote action) tracks whether the stamped tag is an ADK scope tag.
|
|
7522
|
+
if (typeof stampedScopeTag === "string" && stampedScopeTag.length > 0) {
|
|
7523
|
+
return {
|
|
7524
|
+
ok: true,
|
|
7525
|
+
tags: [stampedScopeTag, ...provenance],
|
|
7526
|
+
adkSourced: stampedScopeTag.startsWith(ADK_SCOPE_TAG_PREFIX),
|
|
7527
|
+
};
|
|
7528
|
+
}
|
|
7529
|
+
const adkTags = new Set();
|
|
7530
|
+
let anySourceUnreadable = false;
|
|
7531
|
+
for (const s of sources) {
|
|
7532
|
+
if (!s.ok) {
|
|
7533
|
+
anySourceUnreadable = true;
|
|
7534
|
+
continue;
|
|
7535
|
+
}
|
|
7536
|
+
for (const t of s.tags) {
|
|
7537
|
+
if (typeof t === "string" && t.startsWith(ADK_SCOPE_TAG_PREFIX))
|
|
7538
|
+
adkTags.add(t);
|
|
7539
|
+
}
|
|
7540
|
+
}
|
|
7541
|
+
// No positive ADK evidence → non-ADK. An unreadable source with zero ADK
|
|
7542
|
+
// evidence does NOT fail closed here (that would regress non-ADK promotion);
|
|
7543
|
+
// see the SEAM note above.
|
|
7544
|
+
if (adkTags.size === 0) {
|
|
7545
|
+
return { ok: true, tags: provenance, adkSourced: false };
|
|
7546
|
+
}
|
|
7547
|
+
if (adkTags.size > 1) {
|
|
7548
|
+
return {
|
|
7549
|
+
ok: false,
|
|
7550
|
+
reason: `ADK-sourced candidate spans multiple scope tags (${[...adkTags].sort().join(", ")}); refusing to promote — a merged cross-user claim would leak across users`,
|
|
7551
|
+
};
|
|
7552
|
+
}
|
|
7553
|
+
if (anySourceUnreadable) {
|
|
7554
|
+
return {
|
|
7555
|
+
ok: false,
|
|
7556
|
+
reason: `ADK-sourced candidate has unreadable source memories; the per-user scope tag cannot be confirmed — refusing to promote (fail-closed)`,
|
|
7557
|
+
};
|
|
7558
|
+
}
|
|
7559
|
+
const scopeTag = [...adkTags][0];
|
|
7560
|
+
return { ok: true, tags: [scopeTag, ...provenance], adkSourced: true };
|
|
7561
|
+
}
|
|
7562
|
+
// ─── Machine reviewer namespace (#1205 slice 1205a — Sherlock security req 4) ─
|
|
7563
|
+
// A promotion records a reviewerId that feeds audit/attribution
|
|
7564
|
+
// (schemas/memory.graphql:209). An automated (machine-driven) promotion path
|
|
7565
|
+
// must record a reviewerId that can NEVER be mistaken for a human/agent
|
|
7566
|
+
// reviewer, so attribution isn't laundered. Reserve the `machine:` namespace
|
|
7567
|
+
// for that, and forbid the human `--reviewer` path from claiming it.
|
|
7568
|
+
export const MACHINE_REVIEWER_PREFIX = "machine:";
|
|
7569
|
+
/** Canonical machine reviewerId for the ADK auto-promote consumer (#1205b). */
|
|
7570
|
+
export const MACHINE_REVIEWER_ADK_AUTO_PROMOTE = "machine:adk-auto-promote";
|
|
7571
|
+
/** True iff `id` is in the reserved machine-reviewer namespace — i.e. it
|
|
7572
|
+
* denotes an automated path, not a human or agent reviewer. */
|
|
7573
|
+
export function isMachineReviewerId(id) {
|
|
7574
|
+
return typeof id === "string" && id.startsWith(MACHINE_REVIEWER_PREFIX);
|
|
7575
|
+
}
|
|
7576
|
+
/** The human `flair rem promote` path must not record a reviewerId in the
|
|
7577
|
+
* reserved machine namespace — that would launder automated attribution onto
|
|
7578
|
+
* a human-operated promotion. Returns an error string, or null if allowed. */
|
|
7579
|
+
export function validateHumanReviewerId(reviewerId) {
|
|
7580
|
+
if (isMachineReviewerId(reviewerId)) {
|
|
7581
|
+
return `--reviewer '${reviewerId}' uses the reserved '${MACHINE_REVIEWER_PREFIX}' namespace (reserved for automated promotion); use a human/agent reviewer id`;
|
|
7582
|
+
}
|
|
7583
|
+
return null;
|
|
7584
|
+
}
|
|
7458
7585
|
// ─── flair rem promote ───────────────────────────────────────────────────────
|
|
7459
7586
|
// Slice 2 of FLAIR-NIGHTLY-REM (ops-2qq). Promote a candidate to either Soul
|
|
7460
7587
|
// or persistent Memory. Both --rationale and --to are required (spec § 5: no
|
|
@@ -7482,6 +7609,13 @@ rem
|
|
|
7482
7609
|
process.exit(1);
|
|
7483
7610
|
}
|
|
7484
7611
|
const reviewerId = opts.reviewer || process.env.FLAIR_AGENT_ID || "admin";
|
|
7612
|
+
// The human promote path must not record a reserved machine reviewerId
|
|
7613
|
+
// (Sherlock #4): that would launder automated attribution.
|
|
7614
|
+
const reviewerErr = validateHumanReviewerId(reviewerId);
|
|
7615
|
+
if (reviewerErr) {
|
|
7616
|
+
console.error(`Error: ${reviewerErr}`);
|
|
7617
|
+
process.exit(1);
|
|
7618
|
+
}
|
|
7485
7619
|
try {
|
|
7486
7620
|
// Fetch the candidate
|
|
7487
7621
|
const candidate = await api("GET", `/MemoryCandidate/${encodeURIComponent(candidateId)}`);
|
|
@@ -7492,6 +7626,51 @@ rem
|
|
|
7492
7626
|
console.error(`Error: candidate ${candidateId} ${msg}`);
|
|
7493
7627
|
process.exit(1);
|
|
7494
7628
|
}
|
|
7629
|
+
// ADK tag-lineage: derive the promoted-claim tag set.
|
|
7630
|
+
//
|
|
7631
|
+
// #1205b-1: if the engine stamped an authoritative `scopeTag` on the
|
|
7632
|
+
// candidate (scope:"tagged" distillation), consume it DIRECTLY and skip
|
|
7633
|
+
// the source re-read — correctness no longer depends on the source
|
|
7634
|
+
// memories still being readable (the #1205a seam closure). We only fall
|
|
7635
|
+
// back to re-reading sources when there is NO stamp (a pre-#1205b
|
|
7636
|
+
// candidate, or a non-tagged distillation).
|
|
7637
|
+
const stampedScopeTag = typeof candidate.scopeTag === "string" && candidate.scopeTag.length > 0 ? candidate.scopeTag : undefined;
|
|
7638
|
+
const sourceFetches = [];
|
|
7639
|
+
if (!stampedScopeTag) {
|
|
7640
|
+
// No authoritative stamp — re-read sources to classify. Fail-closed for
|
|
7641
|
+
// ADK-sourced candidates whose per-user scope tag can't be confirmed;
|
|
7642
|
+
// unchanged for non-ADK candidates. See derivePromotedTags for rules.
|
|
7643
|
+
const sourceIds = Array.isArray(candidate.sourceMemoryIds) ? candidate.sourceMemoryIds : [];
|
|
7644
|
+
for (const sid of sourceIds) {
|
|
7645
|
+
try {
|
|
7646
|
+
const mem = await api("GET", `/Memory/${encodeURIComponent(String(sid))}`);
|
|
7647
|
+
if (mem && !mem.error) {
|
|
7648
|
+
sourceFetches.push({ ok: true, tags: Array.isArray(mem.tags) ? mem.tags : [] });
|
|
7649
|
+
}
|
|
7650
|
+
else {
|
|
7651
|
+
sourceFetches.push({ ok: false });
|
|
7652
|
+
}
|
|
7653
|
+
}
|
|
7654
|
+
catch {
|
|
7655
|
+
sourceFetches.push({ ok: false });
|
|
7656
|
+
}
|
|
7657
|
+
}
|
|
7658
|
+
}
|
|
7659
|
+
const tagDecision = derivePromotedTags(candidateId, sourceFetches, stampedScopeTag);
|
|
7660
|
+
if (!tagDecision.ok) {
|
|
7661
|
+
console.error(`Error: candidate ${candidateId} — ${tagDecision.reason}`);
|
|
7662
|
+
process.exit(1);
|
|
7663
|
+
}
|
|
7664
|
+
// Soul entries are agentId-scoped and cannot carry a per-user scope tag,
|
|
7665
|
+
// so an ADK-sourced candidate promoted to Soul is a cross-user leak by
|
|
7666
|
+
// construction — fail closed. (Server-side trust-tier enforcement that
|
|
7667
|
+
// hard-locks the target is the engine slice #1205b; this is the CLI-side
|
|
7668
|
+
// foundation.)
|
|
7669
|
+
if (opts.to === "soul" && tagDecision.adkSourced) {
|
|
7670
|
+
console.error(`Error: candidate ${candidateId} is ADK-sourced (scope tag ${tagDecision.tags[0]}); Soul is agentId-scoped and cannot carry a per-user scope tag — refusing to promote to Soul (would leak across users). Promote ADK-sourced candidates to memory.`);
|
|
7671
|
+
process.exit(1);
|
|
7672
|
+
}
|
|
7673
|
+
const promotedTags = tagDecision.tags;
|
|
7495
7674
|
const decidedAt = new Date().toISOString();
|
|
7496
7675
|
// Write the resulting Soul or Memory entry
|
|
7497
7676
|
if (opts.to === "memory") {
|
|
@@ -7501,7 +7680,7 @@ rem
|
|
|
7501
7680
|
agentId: candidate.agentId,
|
|
7502
7681
|
content: candidate.claim,
|
|
7503
7682
|
durability: "persistent",
|
|
7504
|
-
tags:
|
|
7683
|
+
tags: promotedTags,
|
|
7505
7684
|
derivedFrom: candidate.sourceMemoryIds ?? [],
|
|
7506
7685
|
promotionStatus: "approved",
|
|
7507
7686
|
promotedAt: decidedAt,
|
|
@@ -12758,7 +12937,14 @@ program
|
|
|
12758
12937
|
// a real user query, so a high score means "recall is functioning", not
|
|
12759
12938
|
// "recall is optimal". Its job is to catch recall CRATERING (embeddings
|
|
12760
12939
|
// down, index busted) — the score collapsing toward 0 is the signal, not
|
|
12761
|
-
// fine-grained precision grading.
|
|
12940
|
+
// fine-grained precision grading. NOTE (#1216): this cue-from-the-memory
|
|
12941
|
+
// design is self-polluting as a recall-QUALITY metric — relevance is
|
|
12942
|
+
// query/corpus overlap by construction, so near-duplicate density reads as a
|
|
12943
|
+
// recall collapse (flair#967 / #857 / #996). It is deliberately NOT the
|
|
12944
|
+
// recall-quality number; that authority is the deterministic, fixed-label,
|
|
12945
|
+
// CI-gated eval at test/bench/recall-eval (recall@k / nDCG@10 / MRR). This
|
|
12946
|
+
// probe stays scoped to live-health cratering only. Requires an actual
|
|
12947
|
+
// agent identity to
|
|
12762
12948
|
// query AS (semantic search is agent-scoped) — no identity, fewer than the
|
|
12763
12949
|
// sample-size memories to sample, or a search error all degrade to `null` +
|
|
12764
12950
|
// a `gaps` entry, same graceful-degradation contract as every metric here —
|
package/dist/rem/runner.js
CHANGED
|
@@ -7,7 +7,11 @@
|
|
|
7
7
|
* 3. Maintenance — delegate to /MemoryMaintenance (same code path `flair rem light` uses).
|
|
8
8
|
* 4. Trust-tier filter on input memories — permanently deferred (see below).
|
|
9
9
|
* 5. Distillation — call /ReflectMemories with execute:true, persist staged
|
|
10
|
-
* candidate ids to the audit row.
|
|
10
|
+
* candidate ids to the audit row. TAG-AWARE (#1205b-1): enumerate the
|
|
11
|
+
* active adk:<app>:<user> tags for the agentId and distill ONCE PER TAG
|
|
12
|
+
* under scope:"tagged" (per-user isolation, no cross-user bleed); fall
|
|
13
|
+
* back to the agentId-only scope:"recent" distill when there are no adk:
|
|
14
|
+
* tags (ordinary single-tenant agents, unchanged).
|
|
11
15
|
* 6. Instance-wide dedup-cluster stat — call /MemoryDedupStats (flair-
|
|
12
16
|
* quality Slice 1c). NOT part of FLAIR-NIGHTLY-REM's original per-agent
|
|
13
17
|
* § 4 list — added because a near-duplicate cluster count is inherently
|
|
@@ -54,6 +58,34 @@ import { homedir } from "node:os";
|
|
|
54
58
|
import { createSnapshot } from "./snapshot.js";
|
|
55
59
|
export const REM_PAUSE_FLAG = resolve(homedir(), ".flair", "rem.paused");
|
|
56
60
|
export const REM_NIGHTLY_LOG = resolve(homedir(), ".flair", "logs", "rem-nightly.jsonl");
|
|
61
|
+
// ─── ADK per-tag distillation (#1205b-1) ─────────────────────────────────────
|
|
62
|
+
// adk-flair collapses (app_name, user_id) → ONE Flair agentId, distinguishing
|
|
63
|
+
// users ONLY by a per-user tag `adk:<app>:<user>` (see the adk-flair
|
|
64
|
+
// memory_service compound-tag). An agentId-wide distill (scope:"recent")
|
|
65
|
+
// therefore mixes every user's sessions into shared claims — the cross-user
|
|
66
|
+
// bleed #1205 fixes. The tag-aware cycle instead distills once per active
|
|
67
|
+
// adk: tag under scope:"tagged", so each user's claims come only from that
|
|
68
|
+
// user's sessions.
|
|
69
|
+
export const ADK_TAG_PREFIX = "adk:";
|
|
70
|
+
/**
|
|
71
|
+
* Recency window (ms) used to decide which adk: tags are ACTIVE — a tag is
|
|
72
|
+
* enumerated (and distilled) only if it has memory records created within this
|
|
73
|
+
* window. This is BOTH the bound that keeps enumeration off a full-table scan
|
|
74
|
+
* (the query filters on the indexed `createdAt`) AND the threshold-gate that
|
|
75
|
+
* skips idle users for free (Kern 1a). 48h (not 24h) so a single missed
|
|
76
|
+
* nightly cycle doesn't skip a user who was active only in the gap; distilling
|
|
77
|
+
* a tag pulls ALL its memories regardless, so a skipped cycle only delays, it
|
|
78
|
+
* never loses content.
|
|
79
|
+
*/
|
|
80
|
+
export const DEFAULT_DISTILL_LOOKBACK_MS = 48 * 3600_000;
|
|
81
|
+
/**
|
|
82
|
+
* Per-cycle ceiling on tag-driven /ReflectMemories calls (one LLM call each),
|
|
83
|
+
* so a burst of active ADK users can't starve the nightly window for other
|
|
84
|
+
* agents (Kern 1b). Sequential, not concurrent — matches the runner's existing
|
|
85
|
+
* single-threaded shape. If more tags are active than this, the overflow is
|
|
86
|
+
* recorded in `errors` and picked up on subsequent cycles (they stay active).
|
|
87
|
+
*/
|
|
88
|
+
export const DEFAULT_MAX_TAGS_PER_CYCLE = 200;
|
|
57
89
|
function readPauseSentinel(path) {
|
|
58
90
|
try {
|
|
59
91
|
const contents = readFileSync(path, "utf-8");
|
|
@@ -134,6 +166,59 @@ async function fetchPendingCandidateCount(api, agentId) {
|
|
|
134
166
|
return 0;
|
|
135
167
|
}
|
|
136
168
|
}
|
|
169
|
+
/**
|
|
170
|
+
* Derive the DISTINCT active `adk:<app>:<user>` tags from an agent's memory
|
|
171
|
+
* set (#1205b-1). "Active" = the memory was created at/after `sinceDate`.
|
|
172
|
+
* Returns the distinct set, sorted for determinism.
|
|
173
|
+
*
|
|
174
|
+
* Operates over the memories the runner ALREADY fetched for the snapshot
|
|
175
|
+
* (step 2's `GET /Memory?agentId=<id>`), so enumeration adds NO extra query
|
|
176
|
+
* and NO additional table scan on top of what the cycle already does — a
|
|
177
|
+
* separate bounded distinct-tag DB query is NOT available here: the Memory
|
|
178
|
+
* resource exposes no REST `search_by_conditions` handler (that verb 405s —
|
|
179
|
+
* only MemoryCandidate overrides it), and Harper's ops-API `search_by_
|
|
180
|
+
* conditions` needs admin creds the agent-authed nightly runner doesn't carry.
|
|
181
|
+
* Reusing the already-loaded set is cheaper than either (no second round-trip)
|
|
182
|
+
* and sidesteps that seam entirely. See the module header + issue #1205.
|
|
183
|
+
*
|
|
184
|
+
* The recency cutoff is applied in-memory here as the threshold-gate (Kern 1a):
|
|
185
|
+
* a tag with no records at/after `sinceDate` is idle and is skipped for free.
|
|
186
|
+
* Distilling a tag later still pulls ALL its memories (scope:"tagged" ignores
|
|
187
|
+
* recency), so a skipped cycle only delays, never loses.
|
|
188
|
+
*
|
|
189
|
+
* OWNER-ONLY (Sherlock's user-enumeration-oracle flag): tags are collected
|
|
190
|
+
* ONLY from records whose `agentId` equals `agentId` (the runner's own id).
|
|
191
|
+
* This is load-bearing, NOT belt-and-suspenders: the snapshot fetch
|
|
192
|
+
* (`GET /Memory?agentId=<id>`) resolves through Memory's "open-within-org"
|
|
193
|
+
* read scope and returns org-wide rows — the `?agentId=` query param is NOT an
|
|
194
|
+
* owner filter (verified empirically against Harper, #1205b-1). Without this
|
|
195
|
+
* per-record agentId check the runner would enumerate (and try to distill)
|
|
196
|
+
* OTHER agents' adk tags — a cross-agent user-enumeration oracle. Filtering
|
|
197
|
+
* here confines enumeration to the runner's own users, and the reduction is
|
|
198
|
+
* pure in-process (no endpoint an attacker could point at another id).
|
|
199
|
+
*/
|
|
200
|
+
export function deriveActiveAdkTags(memories, sinceDate, agentId) {
|
|
201
|
+
const tags = new Set();
|
|
202
|
+
for (const m of memories) {
|
|
203
|
+
if (!m || typeof m !== "object")
|
|
204
|
+
continue;
|
|
205
|
+
// Owner scope: only this agent's own records (the fetch is org-wide).
|
|
206
|
+
if (m.agentId !== agentId)
|
|
207
|
+
continue;
|
|
208
|
+
const createdAt = m.createdAt;
|
|
209
|
+
// Recency / threshold gate: skip idle tags (no record since the cutoff).
|
|
210
|
+
if (!createdAt || new Date(createdAt) < sinceDate)
|
|
211
|
+
continue;
|
|
212
|
+
const mt = m.tags;
|
|
213
|
+
if (!Array.isArray(mt))
|
|
214
|
+
continue;
|
|
215
|
+
for (const t of mt) {
|
|
216
|
+
if (typeof t === "string" && t.startsWith(ADK_TAG_PREFIX))
|
|
217
|
+
tags.add(t);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
return [...tags].sort();
|
|
221
|
+
}
|
|
137
222
|
/**
|
|
138
223
|
* Runs one nightly cycle for the given agent. See module header for steps.
|
|
139
224
|
* Pure orchestration; all I/O goes through injected dependencies.
|
|
@@ -167,10 +252,14 @@ export async function runNightlyCycle(opts) {
|
|
|
167
252
|
let memoryCount = 0;
|
|
168
253
|
let soulCount = 0;
|
|
169
254
|
let pendingCandidates = 0;
|
|
255
|
+
// #1205b-1: the memories fetched here (for the snapshot) are reused by the
|
|
256
|
+
// step-5 tag enumeration — no second fetch. Hoisted so step 5 can read them.
|
|
257
|
+
let fetchedMemories = [];
|
|
170
258
|
try {
|
|
171
259
|
// Fetch agent data
|
|
172
260
|
const memoriesRaw = await opts.apiCall("GET", `/Memory?agentId=${encodeURIComponent(opts.agentId)}`);
|
|
173
261
|
const memories = asArray(memoriesRaw);
|
|
262
|
+
fetchedMemories = memories;
|
|
174
263
|
memoryCount = memories.length;
|
|
175
264
|
const soulRaw = await opts.apiCall("GET", `/Soul?agentId=${encodeURIComponent(opts.agentId)}`);
|
|
176
265
|
const souls = asArray(soulRaw);
|
|
@@ -251,34 +340,89 @@ export async function runNightlyCycle(opts) {
|
|
|
251
340
|
// the same way dryRun skips the snapshot write.
|
|
252
341
|
// When the call IS attempted (success or failure), the audit row's `slice`
|
|
253
342
|
// flips to "2" — "2-maintenance" is reserved for the dry-run skip case.
|
|
343
|
+
//
|
|
344
|
+
// #1205b-1 — TAG-AWARE distillation. First enumerate the active
|
|
345
|
+
// adk:<app>:<user> tags for this agentId. If any exist, this is (or includes)
|
|
346
|
+
// an ADK agentId whose users share one agentId and are separated ONLY by
|
|
347
|
+
// tag: distill ONCE PER TAG under scope:"tagged" so each user's candidates
|
|
348
|
+
// are distilled from that user's sessions alone (no cross-user bleed). If
|
|
349
|
+
// NONE exist, this is an ordinary single-tenant agent — fall back to the
|
|
350
|
+
// unchanged agentId-only scope:"recent" distill so non-ADK agents behave
|
|
351
|
+
// exactly as before. The single-node-runs-the-timer property is unchanged:
|
|
352
|
+
// this all runs inside the one cycle on the one node.
|
|
254
353
|
let candidates;
|
|
354
|
+
const collectStagedIds = (obj) => asArray(obj.candidates)
|
|
355
|
+
.map((c) => (c && typeof c === "object" ? c.id : c))
|
|
356
|
+
.filter((id) => typeof id === "string");
|
|
255
357
|
if (!opts.dryRun) {
|
|
256
358
|
sliceLabel = "2";
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
359
|
+
const distillSince = opts.distillSince ?? new Date(startedMs - DEFAULT_DISTILL_LOOKBACK_MS);
|
|
360
|
+
const maxTags = opts.maxTagsPerCycle ?? DEFAULT_MAX_TAGS_PER_CYCLE;
|
|
361
|
+
// Derive active adk: tags from the memories already fetched in step 2 — no
|
|
362
|
+
// extra query/scan. See deriveActiveAdkTags for why a separate bounded DB
|
|
363
|
+
// query isn't available (Memory has no REST search handler; ops-API needs
|
|
364
|
+
// admin the runner lacks).
|
|
365
|
+
const activeAdkTags = deriveActiveAdkTags(fetchedMemories, distillSince, opts.agentId);
|
|
366
|
+
if (activeAdkTags.length > 0) {
|
|
367
|
+
// Per-tag distillation path (ADK). One scope:"tagged" call per active
|
|
368
|
+
// tag; aggregate every batch's staged ids into the single `candidates`
|
|
369
|
+
// list. A per-tag failure is recorded and does NOT abort the remaining
|
|
370
|
+
// tags (or the cycle) — the same non-fatal discipline the agentId-only
|
|
371
|
+
// path uses below.
|
|
372
|
+
const staged = [];
|
|
373
|
+
const tagsToRun = activeAdkTags.slice(0, maxTags);
|
|
374
|
+
if (activeAdkTags.length > maxTags) {
|
|
375
|
+
errors.push(`distillation: ${activeAdkTags.length} active adk tags exceed the per-cycle cap (${maxTags}); ${activeAdkTags.length - maxTags} deferred to a later cycle`);
|
|
268
376
|
}
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
377
|
+
for (const tag of tagsToRun) {
|
|
378
|
+
try {
|
|
379
|
+
const reflectRaw = await opts.apiCall("POST", "/ReflectMemories", {
|
|
380
|
+
agentId: opts.agentId,
|
|
381
|
+
execute: true,
|
|
382
|
+
scope: "tagged",
|
|
383
|
+
tag,
|
|
384
|
+
});
|
|
385
|
+
const obj = (reflectRaw && typeof reflectRaw === "object") ? reflectRaw : {};
|
|
386
|
+
if (obj.error) {
|
|
387
|
+
errors.push(`distillation[${tag}]: ${describeApiError(obj.error)}`);
|
|
388
|
+
}
|
|
389
|
+
else {
|
|
390
|
+
staged.push(...collectStagedIds(obj));
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
catch (err) {
|
|
394
|
+
errors.push(`distillation[${tag}]: ${describeApiError(err?.message ?? err)}`);
|
|
395
|
+
}
|
|
274
396
|
}
|
|
397
|
+
// `candidates` is defined (even if empty) whenever distillation was
|
|
398
|
+
// ATTEMPTED this cycle — same contract as the agentId-only path.
|
|
399
|
+
candidates = staged;
|
|
275
400
|
}
|
|
276
|
-
|
|
277
|
-
//
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
401
|
+
else {
|
|
402
|
+
// AgentId-only path (non-ADK, unchanged pre-#1205b behavior).
|
|
403
|
+
try {
|
|
404
|
+
const reflectRaw = await opts.apiCall("POST", "/ReflectMemories", {
|
|
405
|
+
agentId: opts.agentId,
|
|
406
|
+
execute: true,
|
|
407
|
+
});
|
|
408
|
+
const obj = (reflectRaw && typeof reflectRaw === "object") ? reflectRaw : {};
|
|
409
|
+
if (obj.error) {
|
|
410
|
+
// Defensive: a 200 response shouldn't carry { error }, since
|
|
411
|
+
// MemoryReflect signals failure via HTTP status (503/502) — apiCall
|
|
412
|
+
// implementations throw for those. Handled the same way regardless.
|
|
413
|
+
errors.push(`distillation: ${describeApiError(obj.error)}`);
|
|
414
|
+
}
|
|
415
|
+
else {
|
|
416
|
+
candidates = collectStagedIds(obj);
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
catch (err) {
|
|
420
|
+
// Distillation failure is recorded, not fatal — maintenance already
|
|
421
|
+
// succeeded and the cycle's guaranteed steps are done (spec § 3B item
|
|
422
|
+
// 3). Zero partial candidates is guaranteed server-side (all-or-
|
|
423
|
+
// nothing staging in /ReflectMemories).
|
|
424
|
+
errors.push(`distillation: ${describeApiError(err?.message ?? err)}`);
|
|
425
|
+
}
|
|
282
426
|
}
|
|
283
427
|
}
|
|
284
428
|
// Step 6 (flair-quality Slice 1c): instance-wide dedup-cluster stat.
|