@tpsdev-ai/flair 0.28.0 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/bridges/builtins/index.js +1 -1
- package/dist/bridges/discover.js +1 -1
- package/dist/bridges/runtime/load-plugin.js +1 -1
- package/dist/bridges/scaffold.js +3 -3
- package/dist/bridges/types.js +1 -1
- package/dist/cli-shim.cjs +1 -1
- package/dist/cli.js +357 -130
- package/dist/deploy.js +29 -19
- package/dist/fabric-upgrade.js +61 -23
- package/dist/fleet-verify.js +3 -3
- package/dist/lib/mcp-enable.js +2 -2
- package/dist/rem/runner.js +2 -2
- package/dist/rem/scheduler.js +226 -17
- package/dist/resources/A2AAdapter.js +1 -1
- package/dist/resources/Admin.js +1 -1
- package/dist/resources/AdminConnectors.js +1 -1
- package/dist/resources/AdminDashboard.js +1 -1
- package/dist/resources/AdminIdp.js +1 -1
- package/dist/resources/AdminInstance.js +1 -1
- package/dist/resources/AdminMemory.js +1 -1
- package/dist/resources/AdminPrincipals.js +1 -1
- package/dist/resources/Agent.js +1 -1
- package/dist/resources/AgentCard.js +1 -1
- package/dist/resources/AgentSeed.js +1 -1
- package/dist/resources/AttentionQuery.js +1 -1
- package/dist/resources/Credential.js +1 -1
- package/dist/resources/Federation.js +1 -1
- package/dist/resources/Instance.js +1 -1
- package/dist/resources/Integration.js +1 -1
- package/dist/resources/MCPClientMetadata.js +1 -1
- package/dist/resources/Memory.js +1 -1
- package/dist/resources/MemoryBootstrap.js +1 -1
- package/dist/resources/MemoryCandidate.js +168 -0
- package/dist/resources/MemoryConsolidate.js +1 -1
- package/dist/resources/MemoryDedupStats.js +1 -1
- package/dist/resources/MemoryFeed.js +1 -1
- package/dist/resources/MemoryGrant.js +1 -1
- package/dist/resources/MemoryMaintenance.js +1 -1
- package/dist/resources/MemoryReflect.js +3 -3
- package/dist/resources/MemoryReindex.js +1 -1
- package/dist/resources/MemoryUsage.js +1 -1
- package/dist/resources/OAuth.js +1 -1
- package/dist/resources/OAuthClient.js +1 -1
- package/dist/resources/OrgEvent.js +1 -1
- package/dist/resources/OrgEventCatchup.js +1 -1
- package/dist/resources/OrgEventMaintenance.js +1 -1
- package/dist/resources/PairingToken.js +1 -1
- package/dist/resources/Peer.js +1 -1
- package/dist/resources/Presence.js +3 -3
- package/dist/resources/RecordUsage.js +16 -2
- package/dist/resources/Relationship.js +1 -1
- package/dist/resources/SemanticSearch.js +1 -1
- package/dist/resources/SkillScan.js +1 -1
- package/dist/resources/Soul.js +1 -1
- package/dist/resources/SoulFeed.js +1 -1
- package/dist/resources/WorkspaceLatest.js +1 -1
- package/dist/resources/WorkspaceState.js +1 -1
- package/dist/resources/XAA.js +1 -1
- package/dist/resources/a2a-url.js +1 -1
- package/dist/resources/agent-auth.js +1 -1
- package/dist/resources/agentcard-fields.js +1 -1
- package/dist/resources/auth-middleware.js +1 -1
- package/dist/resources/embeddings-boot.js +4 -4
- package/dist/resources/embeddings-provider.js +13 -43
- package/dist/resources/federation-cleanup.js +5 -5
- package/dist/resources/federation-nonce-store.js +5 -5
- package/dist/resources/health.js +29 -6
- package/dist/resources/instance-identity.js +1 -1
- package/dist/resources/mcp-client-metadata-fields.js +1 -1
- package/dist/resources/mcp-handler.js +1 -1
- package/dist/resources/mcp-oauth.js +2 -2
- package/dist/resources/mcp-tools.js +9 -1
- package/dist/resources/memory-read-scope.js +1 -1
- package/dist/resources/memory-reflect-lib.js +4 -4
- package/dist/resources/memory-visibility.js +3 -3
- package/dist/resources/migration-boot.js +1 -1
- package/dist/resources/migrations/embedding-stamp.js +3 -3
- package/dist/resources/migrations/graph-heal.js +2 -2
- package/dist/resources/migrations/ledger.js +1 -1
- package/dist/resources/migrations/registry.js +14 -5
- package/dist/resources/migrations/runner.js +64 -11
- package/dist/resources/migrations/synthetic-test-migration.js +1 -1
- package/dist/resources/migrations/visibility-backfill.js +232 -0
- package/dist/resources/models-dir.js +55 -0
- package/dist/resources/record-types.js +30 -2
- package/dist/resources/rerank-provider.js +24 -4
- package/dist/resources/semantic-retrieval-core.js +1 -1
- package/dist/resources/usage-recording.js +73 -7
- package/dist/resources/version.js +33 -0
- package/dist/src/rem/scheduler.js +449 -0
- package/docs/assets/flair-cross-orchestrator.cast +1 -1
- package/docs/bridges.md +3 -3
- package/docs/deployment.md +1 -0
- package/docs/mcp-clients.md +3 -1
- package/docs/n8n.md +1 -2
- package/docs/notes/rem-ux.md +1 -1
- package/docs/rem.md +2 -0
- package/docs/supply-chain-policy.md +4 -4
- package/docs/upgrade.md +2 -2
- package/package.json +4 -4
- package/schemas/memory.graphql +27 -12
|
@@ -36,11 +36,11 @@
|
|
|
36
36
|
* what this migration removes: `models.embed` is a process-wide singleton
|
|
37
37
|
* Harper initializes at boot, off the first-call dynamic-import race.
|
|
38
38
|
*
|
|
39
|
-
*
|
|
39
|
+
* `harper` is dynamic-imported here (deferred to first actual
|
|
40
40
|
* getEmbedding()/getMode()/getStatus() call), NOT statically imported like
|
|
41
41
|
* every other resource's `Resource`/`databases`/`server` — those work
|
|
42
42
|
* unmocked-import-free only because every test that (transitively) imports
|
|
43
|
-
* them already mocks
|
|
43
|
+
* them already mocks `harper` first. Statically importing
|
|
44
44
|
* `models` here would make *any* test that imports this file for
|
|
45
45
|
* `resolveModelsDir()` alone (test/unit/embeddings-models-dir.test.ts, which
|
|
46
46
|
* has no reason to know about Harper at all) also eagerly load Harper's real
|
|
@@ -107,9 +107,6 @@
|
|
|
107
107
|
* still a separate, deliberate step from shipping this code — see the PR
|
|
108
108
|
* description for that distinction.
|
|
109
109
|
*/
|
|
110
|
-
import { join } from "node:path";
|
|
111
|
-
import { existsSync } from "node:fs";
|
|
112
|
-
import { homedir } from "node:os";
|
|
113
110
|
const VALID_INPUT_TYPES = new Set(["document", "query"]);
|
|
114
111
|
/**
|
|
115
112
|
* THE GATE (flair#504 Phase 2 — flipped ON). A module-level constant, not an
|
|
@@ -189,7 +186,7 @@ function prefixesEnabled() {
|
|
|
189
186
|
* Build the options object passed to `models.embed()`. Pulled out as its own
|
|
190
187
|
* pure, harper-free function so the value-forwarding (and the reject-a-
|
|
191
188
|
* wrong-value guard) is unit-testable without touching the deferred
|
|
192
|
-
*
|
|
189
|
+
* `harper` import this file's header explains — see
|
|
193
190
|
* test/unit/embeddings-provider-input-type.test.ts.
|
|
194
191
|
*
|
|
195
192
|
* Gate ON (default): rejects anything other than the literal
|
|
@@ -220,7 +217,7 @@ let _modelsApi;
|
|
|
220
217
|
/**
|
|
221
218
|
* Resolve (and cache) Harper's `models` facade via a deferred import — see file header.
|
|
222
219
|
*
|
|
223
|
-
* NOTE for anyone using this as a reference: `models` (the
|
|
220
|
+
* NOTE for anyone using this as a reference: `models` (the `harper`
|
|
224
221
|
* package export) and a component's `scope.models` are the SAME boot-time
|
|
225
222
|
* singleton — Harper's jsLoader hands components `scope.models = <the global
|
|
226
223
|
* models>` (two accessors, one object; the model registry lives on that singleton).
|
|
@@ -232,49 +229,22 @@ let _modelsApi;
|
|
|
232
229
|
async function getModelsApi() {
|
|
233
230
|
if (_modelsApi)
|
|
234
231
|
return _modelsApi;
|
|
235
|
-
const harper = await import("
|
|
232
|
+
const harper = await import("harper");
|
|
236
233
|
_modelsApi = harper.models;
|
|
237
234
|
return _modelsApi;
|
|
238
235
|
}
|
|
239
236
|
/**
|
|
240
237
|
* Resolve the directory the embeddings model lives in / downloads into.
|
|
241
238
|
*
|
|
242
|
-
*
|
|
243
|
-
*
|
|
244
|
-
*
|
|
245
|
-
*
|
|
246
|
-
*
|
|
247
|
-
*
|
|
248
|
-
*
|
|
249
|
-
* downloaded into the package dir before this fix;
|
|
250
|
-
* never used as a download target on fresh installs.
|
|
251
|
-
* 4. ~/.flair/data/models — last-resort default when ROOTPATH is unset.
|
|
252
|
-
*
|
|
253
|
-
* The chosen dir is always writable, so the embeddings engine can download the
|
|
254
|
-
* model on first use without hitting EACCES on a root-owned package dir.
|
|
255
|
-
*
|
|
256
|
-
* Called by `resources/embeddings-boot.ts` (flair#694) to build the
|
|
257
|
-
* `modelsDir` it passes to harper-fabric-embeddings' `register()` — both run
|
|
258
|
-
* in the SAME process now (unlike the retired src/cli.ts-side computation,
|
|
259
|
-
* which duplicated this exact default because it ran in a separate process
|
|
260
|
-
* that set the resulting value via an env var). Still exported and tested
|
|
261
|
-
* independently (test/unit/embeddings-models-dir.test.ts) as the single
|
|
262
|
-
* documented source of truth for this default.
|
|
239
|
+
* Implementation moved to `resources/models-dir.ts` (flair#815) so the
|
|
240
|
+
* reranker can share the exact same resolution without importing this module
|
|
241
|
+
* (several unit-isolated tests `mock.module()` this file wholesale — see
|
|
242
|
+
* models-dir.ts's header). Re-exported here so existing consumers keep their
|
|
243
|
+
* import site: `resources/embeddings-boot.ts` (flair#694), which builds the
|
|
244
|
+
* `modelsDir` it passes to harper-fabric-embeddings' `register()`, and
|
|
245
|
+
* test/unit/embeddings-models-dir.test.ts, which pins the resolution order.
|
|
263
246
|
*/
|
|
264
|
-
export
|
|
265
|
-
const override = process.env.FLAIR_MODELS_DIR;
|
|
266
|
-
if (override)
|
|
267
|
-
return override;
|
|
268
|
-
const rootPath = process.env.ROOTPATH;
|
|
269
|
-
if (rootPath)
|
|
270
|
-
return join(rootPath, "models");
|
|
271
|
-
// Backward compat: a prior (writable) install may have the model cached in
|
|
272
|
-
// the package dir. Reuse it rather than re-downloading — but only if present.
|
|
273
|
-
const cwdModels = join(process.cwd(), "models");
|
|
274
|
-
if (existsSync(cwdModels))
|
|
275
|
-
return cwdModels;
|
|
276
|
-
return join(homedir(), ".flair", "data", "models");
|
|
277
|
-
}
|
|
247
|
+
export { resolveModelsDir } from "./models-dir.js";
|
|
278
248
|
// Kern: "Boot probe MUST have a 5-10s timeout ... on timeout set mode='none'
|
|
279
249
|
// + log a warning. A hung probe ... must never block Harper boot."
|
|
280
250
|
const PROBE_TIMEOUT_MS = 8_000;
|
|
@@ -11,13 +11,13 @@ let cleanupTimer = null;
|
|
|
11
11
|
* and performs housekeeping on expired/unconsumed token records.
|
|
12
12
|
*
|
|
13
13
|
* In test environments, callers pass mock serverOp/db via `opts` so this
|
|
14
|
-
* module never imports
|
|
14
|
+
* module never imports harper at the top level (which would
|
|
15
15
|
* crash when STORAGE_PATH isn't set).
|
|
16
16
|
*/
|
|
17
17
|
export async function initFederationCleanup(opts) {
|
|
18
18
|
const immediate = opts?.immediateTick ?? true;
|
|
19
19
|
// Resolve server / databases: use caller-supplied mocks when available,
|
|
20
|
-
// otherwise lazy-import
|
|
20
|
+
// otherwise lazy-import harper at call time.
|
|
21
21
|
let svr;
|
|
22
22
|
let db;
|
|
23
23
|
if (opts?.serverOp && opts?.db) {
|
|
@@ -26,12 +26,12 @@ export async function initFederationCleanup(opts) {
|
|
|
26
26
|
}
|
|
27
27
|
else {
|
|
28
28
|
try {
|
|
29
|
-
const harper = await import("
|
|
29
|
+
const harper = await import("harper");
|
|
30
30
|
svr = opts?.serverOp ?? harper.server.operation;
|
|
31
31
|
db = opts?.db ?? harper.databases;
|
|
32
32
|
}
|
|
33
33
|
catch (err) {
|
|
34
|
-
console.error("[federation-cleanup] failed to load
|
|
34
|
+
console.error("[federation-cleanup] failed to load harper:", err?.message ?? err);
|
|
35
35
|
return;
|
|
36
36
|
}
|
|
37
37
|
}
|
|
@@ -94,7 +94,7 @@ export async function runCleanupTick(opts = {}) {
|
|
|
94
94
|
db = opts.db;
|
|
95
95
|
}
|
|
96
96
|
else {
|
|
97
|
-
const harper = await import("
|
|
97
|
+
const harper = await import("harper");
|
|
98
98
|
svr = opts.serverOp ?? harper.server.operation;
|
|
99
99
|
db = opts.db ?? harper.databases;
|
|
100
100
|
}
|
|
@@ -44,7 +44,7 @@ async function resolveDb(opts, cache) {
|
|
|
44
44
|
if (opts.db)
|
|
45
45
|
return opts.db;
|
|
46
46
|
if (!cache.db) {
|
|
47
|
-
cache.db = import("
|
|
47
|
+
cache.db = import("harper").then((h) => h.databases);
|
|
48
48
|
}
|
|
49
49
|
return cache.db;
|
|
50
50
|
}
|
|
@@ -123,7 +123,7 @@ export async function runNonceCleanupTick(opts = {}) {
|
|
|
123
123
|
db = opts.db;
|
|
124
124
|
}
|
|
125
125
|
else {
|
|
126
|
-
const harper = await import("
|
|
126
|
+
const harper = await import("harper");
|
|
127
127
|
db = harper.databases;
|
|
128
128
|
}
|
|
129
129
|
const retentionMs = opts.retentionMs ?? DEFAULT_RETENTION_MS;
|
|
@@ -159,7 +159,7 @@ export async function runNonceCleanupTick(opts = {}) {
|
|
|
159
159
|
* Initialise the periodic Nonce-table eviction sweep (5-min cadence).
|
|
160
160
|
*
|
|
161
161
|
* In test environments, callers pass a mock `db` via `opts` so this module
|
|
162
|
-
* never imports
|
|
162
|
+
* never imports harper at the top level (mirrors
|
|
163
163
|
* federation-cleanup.ts's `initFederationCleanup`).
|
|
164
164
|
*/
|
|
165
165
|
export async function initNonceStoreCleanup(opts) {
|
|
@@ -170,11 +170,11 @@ export async function initNonceStoreCleanup(opts) {
|
|
|
170
170
|
}
|
|
171
171
|
else {
|
|
172
172
|
try {
|
|
173
|
-
const harper = await import("
|
|
173
|
+
const harper = await import("harper");
|
|
174
174
|
db = harper.databases;
|
|
175
175
|
}
|
|
176
176
|
catch (err) {
|
|
177
|
-
console.error("[federation-nonce-store] failed to load
|
|
177
|
+
console.error("[federation-nonce-store] failed to load harper:", err?.message ?? err);
|
|
178
178
|
return;
|
|
179
179
|
}
|
|
180
180
|
}
|
package/dist/resources/health.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Resource, databases } from "
|
|
1
|
+
import { Resource, databases } from "harper";
|
|
2
2
|
import { promises as fsp, existsSync, readFileSync } from "node:fs";
|
|
3
3
|
import { homedir, platform } from "node:os";
|
|
4
4
|
import { join, dirname } from "node:path";
|
|
@@ -452,13 +452,33 @@ export class HealthDetail extends Resource {
|
|
|
452
452
|
const lastLightAt = findLast("light");
|
|
453
453
|
const lastRapidAt = findLast("rapid");
|
|
454
454
|
const lastRestorativeAt = findLast("restorative");
|
|
455
|
+
// File presence only proves the scheduler templates were written —
|
|
456
|
+
// NOT that launchctl/systemctl actually loaded the job. A
|
|
457
|
+
// `systemctl --user enable --now` that fails (e.g. no session bus)
|
|
458
|
+
// still leaves the timer file on disk, so file-existence alone would
|
|
459
|
+
// report "enabled" for a scheduler that has nothing scheduled
|
|
460
|
+
// (flair#850). Query genuine active state once the file is known to
|
|
461
|
+
// exist; async + short-timeout so a slow/hung service manager can't
|
|
462
|
+
// stall this request.
|
|
455
463
|
let nightlyEnabled = null;
|
|
464
|
+
let nightlyInstalled = false;
|
|
456
465
|
const plat = platform();
|
|
457
|
-
if (plat === "darwin") {
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
466
|
+
if (plat === "darwin" || plat === "linux") {
|
|
467
|
+
nightlyInstalled = plat === "darwin"
|
|
468
|
+
? await exists(join(homedir(), "Library", "LaunchAgents", "dev.flair.rem.nightly.plist"))
|
|
469
|
+
: await exists(join(homedir(), ".config", "systemd", "user", "flair-rem-nightly.timer"));
|
|
470
|
+
if (nightlyInstalled) {
|
|
471
|
+
try {
|
|
472
|
+
const { queryActiveStateAsync } = await import("../src/rem/scheduler.js");
|
|
473
|
+
nightlyEnabled = await queryActiveStateAsync(plat);
|
|
474
|
+
}
|
|
475
|
+
catch {
|
|
476
|
+
nightlyEnabled = null;
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
else {
|
|
480
|
+
nightlyEnabled = false;
|
|
481
|
+
}
|
|
462
482
|
}
|
|
463
483
|
const nightlyRecords = await tailJsonl(nightlyLog);
|
|
464
484
|
const lastNightlyRec = nightlyRecords[nightlyRecords.length - 1];
|
|
@@ -496,6 +516,9 @@ export class HealthDetail extends Resource {
|
|
|
496
516
|
if (nightlyEnabled && lastNightlyAt && nowMs - new Date(lastNightlyAt).getTime() > 48 * 3600 * 1000) {
|
|
497
517
|
warnings.push({ level: "warn", message: "nightly REM hasn't run in >48h" });
|
|
498
518
|
}
|
|
519
|
+
if (nightlyInstalled && nightlyEnabled === false) {
|
|
520
|
+
warnings.push({ level: "warn", message: "REM nightly scheduler files are written but not active — nothing is scheduled (see `flair rem nightly status`)" });
|
|
521
|
+
}
|
|
499
522
|
}
|
|
500
523
|
}
|
|
501
524
|
catch {
|
|
@@ -39,7 +39,7 @@
|
|
|
39
39
|
* `allowedHosts` configured → rejected; leaked private `d` in a JWK →
|
|
40
40
|
* rejected by the plugin even if our own build-time guard were bypassed).
|
|
41
41
|
*
|
|
42
|
-
* Kept free of any
|
|
42
|
+
* Kept free of any harper import (mirrors agentcard-fields.ts)
|
|
43
43
|
* so the document shape is unit-testable without spinning up Harper.
|
|
44
44
|
*/
|
|
45
45
|
/** The grant this agent's CIMD document targets — formalized by oauth#161, accepted by the published 2.2.0 validator. See module header. */
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
* Harper HTTP listeners return `{ status, body, headers? }`. MCP messages are
|
|
24
24
|
* JSON-RPC 2.0, so we serialize the JSON-RPC response object as the body.
|
|
25
25
|
*/
|
|
26
|
-
import { databases } from "
|
|
26
|
+
import { databases } from "harper";
|
|
27
27
|
import { randomBytes } from "node:crypto";
|
|
28
28
|
import { TOOLS, listToolDefs } from "./mcp-tools.js";
|
|
29
29
|
// The MCP protocol revision we implement (initialize handshake).
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
* only meaningful when an operator has set the issuer + enabled the flag (see
|
|
18
18
|
* docs). The plugin serves DCR / authorize / token / JWKS / discovery.
|
|
19
19
|
*/
|
|
20
|
-
import * as harper from "
|
|
20
|
+
import * as harper from "harper";
|
|
21
21
|
import { mcpOAuthEnabled, mcpAuthConfig } from "./mcp-oauth-flag.js";
|
|
22
22
|
async function defaultLoadWithMCPAuth() {
|
|
23
23
|
// Dynamic import so the dep is only required when the surface is enabled.
|
|
@@ -53,7 +53,7 @@ export async function registerMcpOAuthRoute(deps = {}) {
|
|
|
53
53
|
const handler = deps.mcpHandler ?? (await import("./mcp-handler.js")).mcpHandler;
|
|
54
54
|
// Read `server` lazily off the namespace (it's a runtime global on the Harper
|
|
55
55
|
// module, not a static named export) so this module links cleanly even where a
|
|
56
|
-
// stub build of
|
|
56
|
+
// stub build of harper lacks the export.
|
|
57
57
|
const srv = deps.server ?? (harper.server);
|
|
58
58
|
// Primary registration: urlPath subroute → own chain (flair's auth-middleware
|
|
59
59
|
// does not run here). `getConfig` pins iss/resource to the AS's values so the
|
|
@@ -35,6 +35,7 @@
|
|
|
35
35
|
* version bump + a new FlairClient method, out of scope for this query-only
|
|
36
36
|
* slice.
|
|
37
37
|
*/
|
|
38
|
+
import { resolveVersion } from "./version.js";
|
|
38
39
|
const H = {};
|
|
39
40
|
const LOADERS = {
|
|
40
41
|
SemanticSearch: async () => (await import("./SemanticSearch.js")).SemanticSearch,
|
|
@@ -248,7 +249,14 @@ async function bootstrap(agent, args) {
|
|
|
248
249
|
// ONLY when requested so a plain bootstrap delegates a byte-identical body.
|
|
249
250
|
if (args?.abstain === true)
|
|
250
251
|
body.abstain = true;
|
|
251
|
-
|
|
252
|
+
// flair#831 — attach the running Flair version to the RESPONSE (not the
|
|
253
|
+
// delegated request body) so the calling agent learns the server version
|
|
254
|
+
// on its very first call.
|
|
255
|
+
const result = unwrap(await h.post(body));
|
|
256
|
+
if (result && typeof result === "object" && !Array.isArray(result)) {
|
|
257
|
+
return { ...result, flairVersion: resolveVersion() };
|
|
258
|
+
}
|
|
259
|
+
return result;
|
|
252
260
|
}
|
|
253
261
|
async function soulSet(agent, args) {
|
|
254
262
|
const Cls = await handler("Soul");
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { databases } from "
|
|
1
|
+
import { databases } from "harper";
|
|
2
2
|
import { PRIVATE_VISIBILITY, isPrivateVisibility } from "./memory-visibility.js";
|
|
3
3
|
/**
|
|
4
4
|
* ─── Centralized Memory read-scoping (the original grant-gated read model → within-org-read-open) ─
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// ─── Memory Reflection — pure logic for /ReflectMemories ────────────────────
|
|
2
2
|
// Pure helpers backing resources/MemoryReflect.ts (FLAIR-NIGHTLY-REM slice 2,
|
|
3
|
-
// §3A — see
|
|
3
|
+
// §3A — see issue #707).
|
|
4
4
|
//
|
|
5
5
|
// Same split as resources/memory-consolidate-lib.ts: importing MemoryReflect.ts
|
|
6
6
|
// pulls in the Harper runtime (`databases`/`Resource`/`models`, storage init),
|
|
@@ -44,7 +44,7 @@ export const GENERATE_TEMPERATURE = 0.2;
|
|
|
44
44
|
//
|
|
45
45
|
// Passed as `responseFormat: { schema: CANDIDATES_SCHEMA }` to models.generate()
|
|
46
46
|
// so backends that honor structured output (Ollama, OpenAI — verified against
|
|
47
|
-
// the pinned
|
|
47
|
+
// the pinned harper 5.1.17's bundled backends) return conformant
|
|
48
48
|
// JSON directly. Not every backend enforces it (Anthropic's Messages API has
|
|
49
49
|
// no equivalent and Harper documents that it silently ignores the option) —
|
|
50
50
|
// this module never trusts the backend to have enforced the schema; every
|
|
@@ -226,7 +226,7 @@ export function parseAndValidateCandidates(raw, gatheredMemoryIds) {
|
|
|
226
226
|
/**
|
|
227
227
|
* Name Harper's models facade sets on the error it throws when no backend is
|
|
228
228
|
* registered for the requested logical name (`ModelBackendNotFoundError`,
|
|
229
|
-
* resources/models/backendRegistry.ts in
|
|
229
|
+
* resources/models/backendRegistry.ts in harper 5.1.17). That
|
|
230
230
|
* class isn't part of the package's public export surface (only its
|
|
231
231
|
* `Models`/type surface is re-exported from the package root), so detecting
|
|
232
232
|
* it here is a documented duck-type on `.name` rather than `instanceof` —
|
|
@@ -237,7 +237,7 @@ export const MODEL_BACKEND_NOT_FOUND_ERROR_NAME = "ModelBackendNotFoundError";
|
|
|
237
237
|
* Calls generate(), validates the result, and on malformed/mismatched output
|
|
238
238
|
* retries exactly once with an explicit `responseFormat: 'json'` (the
|
|
239
239
|
* "json-fallback path" — spec §3A items 2 & 3: build-time check confirmed
|
|
240
|
-
* `GenerateOpts.responseFormat` supports `{ schema }` in
|
|
240
|
+
* `GenerateOpts.responseFormat` supports `{ schema }` in harper
|
|
241
241
|
* 5.1.17's types, but not every backend enforces it, so the first attempt
|
|
242
242
|
* requests schema mode and the fallback attempt requests plain json mode).
|
|
243
243
|
* Both attempts run through the SAME parseAndValidateCandidates — a parse
|
|
@@ -9,18 +9,18 @@
|
|
|
9
9
|
* - src/cli.ts's runFederationSyncOnce() — the federation-sync PUSH filter
|
|
10
10
|
* that must not replicate `private` memories to peer instances.
|
|
11
11
|
*
|
|
12
|
-
* Deliberately has ZERO imports — not even "
|
|
12
|
+
* Deliberately has ZERO imports — not even "harper". That is
|
|
13
13
|
* intentional and load-bearing: src/cli.ts is a standalone CLI entrypoint
|
|
14
14
|
* that runs OUTSIDE any running Harper instance (e.g. `flair federation
|
|
15
15
|
* sync` invoked from a cron/launchd job). resources/memory-read-scope.ts
|
|
16
|
-
* imports `databases` from "
|
|
16
|
+
* imports `databases` from "harper", and that package's
|
|
17
17
|
* top-level init eagerly resolves storage paths and THROWS when there is no
|
|
18
18
|
* live Harper runtime backing it (confirmed empirically — it takes down
|
|
19
19
|
* even `flair --help`). So src/cli.ts must never import
|
|
20
20
|
* resources/memory-read-scope.ts (or anything else that drags that
|
|
21
21
|
* side-effecting import in) directly. This module is the safe seam: a pure
|
|
22
22
|
* function + constant that both sides can import without dragging in
|
|
23
|
-
* "
|
|
23
|
+
* "harper".
|
|
24
24
|
*
|
|
25
25
|
* ── The migration invariant (non-negotiable, mirrors memory-read-scope.ts) ──
|
|
26
26
|
* A record with NO `visibility` field (written before the field existed) is
|
|
@@ -26,7 +26,7 @@
|
|
|
26
26
|
* the `.catch()` below is pure defense-in-depth so a bug there can never
|
|
27
27
|
* take down the process either.
|
|
28
28
|
*/
|
|
29
|
-
import { databases } from "
|
|
29
|
+
import { databases } from "harper";
|
|
30
30
|
import { homedir } from "node:os";
|
|
31
31
|
import { existsSync, readFileSync } from "node:fs";
|
|
32
32
|
import { join, dirname } from "node:path";
|
|
@@ -62,8 +62,8 @@
|
|
|
62
62
|
* embeddings engine to settle before running migrations at all.
|
|
63
63
|
*
|
|
64
64
|
* flair#807 — completion-gate false-negative on 0.10-era / cross-version
|
|
65
|
-
* stores, root-caused against the installed
|
|
66
|
-
* source (node_modules
|
|
65
|
+
* stores, root-caused against the installed `harper@5.1.22`
|
|
66
|
+
* source (node_modules/harper/resources/search.ts): for an
|
|
67
67
|
* INDEXED attribute (`embeddingModel` IS `@indexed` — schemas/memory.graphql),
|
|
68
68
|
* a non-negated `not_equal`/`ne` leaf condition takes `searchByIndex`'s
|
|
69
69
|
* `index && !skipIndex` branch, which reconstructs a SYNTHETIC partial record
|
|
@@ -125,7 +125,7 @@
|
|
|
125
125
|
* resources/migrations/graph-heal.ts). Before enabling any FUTURE bulk
|
|
126
126
|
* re-embed, ship a structural reindex trigger with it.
|
|
127
127
|
*/
|
|
128
|
-
import { databases } from "
|
|
128
|
+
import { databases } from "harper";
|
|
129
129
|
import { getModelId } from "../embeddings-provider.js";
|
|
130
130
|
function defaultMemoryTable() {
|
|
131
131
|
return databases.flair.Memory;
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* every pre-this-change install) against the schema and, because
|
|
10
10
|
* `canonicalizeIndexOptions` does NOT inject defaults (`{type:"HNSW"}` and
|
|
11
11
|
* `{type:"HNSW",M:16}` are DIFFERENT canonical keys — verified against the
|
|
12
|
-
* installed
|
|
12
|
+
* installed harper source, resources/databases.js:
|
|
13
13
|
* canonicalizeIndexOptions + the `indexOptionsChanged` reset of
|
|
14
14
|
* `lastIndexedKey` to undefined), clears the graph store and rebuilds it
|
|
15
15
|
* CLEANLY from the already-correct stored vectors. That rebuild heals the
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
* correctness — it re-PUTs the same vector through the incremental path and
|
|
58
58
|
* does not rebuild the graph.
|
|
59
59
|
*/
|
|
60
|
-
import { databases } from "
|
|
60
|
+
import { databases } from "harper";
|
|
61
61
|
import { existsSync, readFileSync } from "node:fs";
|
|
62
62
|
import { dirname, join } from "node:path";
|
|
63
63
|
import { fileURLToPath } from "node:url";
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
* resources/MemoryReindex.ts's `_reindex` path already rely on for
|
|
18
18
|
* server-internal writes with no HTTP caller behind them.
|
|
19
19
|
*/
|
|
20
|
-
import { databases } from "
|
|
20
|
+
import { databases } from "harper";
|
|
21
21
|
function defaultOrgEventTable() {
|
|
22
22
|
return databases.flair.OrgEvent;
|
|
23
23
|
}
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* registry.ts — the MigrationRegistry: an ordered list of registered
|
|
3
|
-
* migrations. embedding-stamp
|
|
4
|
-
* CI-only variant registers ONLY when
|
|
5
|
-
* true (see synthetic-test-
|
|
3
|
+
* migrations. embedding-stamp, graph-heal, and visibility-backfill always
|
|
4
|
+
* register; the synthetic CI-only variant registers ONLY when
|
|
5
|
+
* shouldRegisterSyntheticMigration() is true (see synthetic-test-
|
|
6
|
+
* migration.ts's doc for the exact gating rule).
|
|
6
7
|
*/
|
|
7
8
|
import { createEmbeddingStampMigration } from "./embedding-stamp.js";
|
|
8
9
|
import { createGraphHealMigration } from "./graph-heal.js";
|
|
10
|
+
import { createVisibilityBackfillMigration } from "./visibility-backfill.js";
|
|
9
11
|
import { createSyntheticTestMigration, shouldRegisterSyntheticMigration } from "./synthetic-test-migration.js";
|
|
10
12
|
export class MigrationRegistry {
|
|
11
13
|
migrations = [];
|
|
@@ -24,17 +26,24 @@ export class MigrationRegistry {
|
|
|
24
26
|
}
|
|
25
27
|
}
|
|
26
28
|
/**
|
|
27
|
-
* Builds the production registry. Always includes embedding-stamp
|
|
28
|
-
* graph-heal; conditionally includes the CI-only
|
|
29
|
+
* Builds the production registry. Always includes embedding-stamp,
|
|
30
|
+
* graph-heal, and visibility-backfill; conditionally includes the CI-only
|
|
31
|
+
* synthetic migration.
|
|
29
32
|
*
|
|
30
33
|
* Order note: graph-heal registers AFTER embedding-stamp so that if a boot
|
|
31
34
|
* ever runs both, the (verify-only) recall check ledgers the state AFTER any
|
|
32
35
|
* re-embed work that cycle already applied — never a stale pre-embed reading.
|
|
36
|
+
* visibility-backfill registers last — it touches an unrelated field
|
|
37
|
+
* (`visibility`, never `embedding`/`embeddingModel`) and has no ordering
|
|
38
|
+
* dependency on either of the other two; appended after them simply as
|
|
39
|
+
* newest-added-last, matching how graph-heal was appended after
|
|
40
|
+
* embedding-stamp.
|
|
33
41
|
*/
|
|
34
42
|
export function buildRegistry(env = process.env) {
|
|
35
43
|
const registry = new MigrationRegistry();
|
|
36
44
|
registry.register(createEmbeddingStampMigration());
|
|
37
45
|
registry.register(createGraphHealMigration());
|
|
46
|
+
registry.register(createVisibilityBackfillMigration());
|
|
38
47
|
if (shouldRegisterSyntheticMigration(env)) {
|
|
39
48
|
registry.register(createSyntheticTestMigration());
|
|
40
49
|
}
|
|
@@ -178,9 +178,11 @@ async function runCycleLocked(deps, lock) {
|
|
|
178
178
|
if (candidates.length === 0) {
|
|
179
179
|
return { ran: false, reason: "nothing pending" };
|
|
180
180
|
}
|
|
181
|
-
// ── ONE shared async pre-hash
|
|
182
|
-
//
|
|
183
|
-
//
|
|
181
|
+
// ── ONE shared async pre-hash at cycle start (K&S: computed after ready,
|
|
182
|
+
// before any migration's first write; failure → halt all pending
|
|
183
|
+
// candidates rather than guessing which ones are safe). Re-baselined
|
|
184
|
+
// below iff an earlier migration's own writes stale it for a later
|
|
185
|
+
// full-envelope gate — see the staleness comment on the loop. ──
|
|
184
186
|
setCyclePhase("pre-hash");
|
|
185
187
|
let envelope;
|
|
186
188
|
try {
|
|
@@ -195,8 +197,49 @@ async function runCycleLocked(deps, lock) {
|
|
|
195
197
|
return { ran: false, reason };
|
|
196
198
|
}
|
|
197
199
|
setCyclePhase("running");
|
|
200
|
+
// The cycle-start envelope is a valid gate BASELINE only until a migration
|
|
201
|
+
// in this same cycle writes rows: a legitimate, already-gated, ledgered
|
|
202
|
+
// write by an EARLIER migration (e.g. visibility-backfill stamping
|
|
203
|
+
// `visibility` — a SOURCE field) would otherwise read as corruption to a
|
|
204
|
+
// later migration's count+full-envelope gate and spuriously halt it
|
|
205
|
+
// ("completion gate failed: rowsRemaining=0, hash envelope mismatch" — the
|
|
206
|
+
// CI-observed failure on PR #845). That gate exists to catch writes the
|
|
207
|
+
// GATED migration didn't declare, not its predecessors' own completed
|
|
208
|
+
// work. So: track staleness conservatively (ANY rows written by any
|
|
209
|
+
// earlier migration, source-field or not — one cheap recompute beats
|
|
210
|
+
// enumerating which fields each migration touches), and re-baseline with a
|
|
211
|
+
// fresh computeCorpusEnvelope immediately BEFORE a full-envelope consumer
|
|
212
|
+
// starts — still before that migration's own first write, which is exactly
|
|
213
|
+
// the invariant-IV wording ("computed before first write and after
|
|
214
|
+
// completion; must match"). Cycles with no full-envelope consumer after a
|
|
215
|
+
// writing migration (every production cycle today — no schema-additive
|
|
216
|
+
// migration is registered outside CI) never pay for a second corpus walk.
|
|
217
|
+
let envelopeStale = false;
|
|
198
218
|
for (const migration of candidates) {
|
|
199
|
-
|
|
219
|
+
if (envelopeStale && postureFor(migration.riskClass).gate === "count+full-envelope") {
|
|
220
|
+
try {
|
|
221
|
+
envelope = await computeCorpusEnvelope(deps.getTable, deps.now);
|
|
222
|
+
envelopeStale = false;
|
|
223
|
+
}
|
|
224
|
+
catch (err) {
|
|
225
|
+
// Same failure class as the cycle-start pre-hash, but scoped: only
|
|
226
|
+
// THIS migration consumes the baseline, so only it halts (the
|
|
227
|
+
// cycle-start variant halts every candidate because nothing has run
|
|
228
|
+
// yet and a failing corpus walk taints them all equally). Halted ≠
|
|
229
|
+
// bricked: it is retried on the next boot's cycle.
|
|
230
|
+
setMigrationProgress({
|
|
231
|
+
id: migration.id,
|
|
232
|
+
rowsDone: 0,
|
|
233
|
+
rowsRemaining: 0,
|
|
234
|
+
state: "halted",
|
|
235
|
+
reason: `pre-flight integrity check failed: ${err?.message ?? String(err)}`,
|
|
236
|
+
});
|
|
237
|
+
continue;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
const wroteRows = await runOneMigration(migration, envelope, deps, lock);
|
|
241
|
+
if (wroteRows)
|
|
242
|
+
envelopeStale = true;
|
|
200
243
|
}
|
|
201
244
|
setCyclePhase("done");
|
|
202
245
|
return { ran: true };
|
|
@@ -243,6 +286,15 @@ async function haltMigration(migration, reason, ctx) {
|
|
|
243
286
|
/* best-effort — the in-memory progress + ledger event already recorded the halt */
|
|
244
287
|
}
|
|
245
288
|
}
|
|
289
|
+
/**
|
|
290
|
+
* Returns whether this migration MAY have written any rows — the cycle
|
|
291
|
+
* loop's envelope-staleness signal (see runCycleLocked). Deliberately
|
|
292
|
+
* conservative: `true` whenever rows were processed OR run() threw
|
|
293
|
+
* mid-batch (a partial batch may have landed before the throw); a false
|
|
294
|
+
* `true` merely costs one extra corpus walk before the next full-envelope
|
|
295
|
+
* gate, while a false `false` would let that gate compare against a stale
|
|
296
|
+
* baseline and spuriously halt.
|
|
297
|
+
*/
|
|
246
298
|
async function runOneMigration(migration, envelope, deps, lock) {
|
|
247
299
|
const startedAt = deps.now().toISOString();
|
|
248
300
|
const state = readMigrationState(deps.statePath);
|
|
@@ -262,7 +314,7 @@ async function runOneMigration(migration, envelope, deps, lock) {
|
|
|
262
314
|
hashEnvelopeMatch: null,
|
|
263
315
|
state,
|
|
264
316
|
});
|
|
265
|
-
return;
|
|
317
|
+
return false; // halted before any write
|
|
266
318
|
}
|
|
267
319
|
const estSnapshot = estimateSnapshotBytes(posture.snapshotScope, initialRemaining);
|
|
268
320
|
const estWorkingSet = estimateWorkingSetBytes(migration.riskClass, initialRemaining);
|
|
@@ -282,7 +334,7 @@ async function runOneMigration(migration, envelope, deps, lock) {
|
|
|
282
334
|
hashEnvelopeMatch: null,
|
|
283
335
|
state,
|
|
284
336
|
});
|
|
285
|
-
return;
|
|
337
|
+
return false; // halted before any write
|
|
286
338
|
}
|
|
287
339
|
// ── Ladder step 3 (+ step 4 fallback): risk-scoped snapshot, or a
|
|
288
340
|
// content-only export if the snapshot mechanism itself fails. ──
|
|
@@ -307,7 +359,7 @@ async function runOneMigration(migration, envelope, deps, lock) {
|
|
|
307
359
|
}
|
|
308
360
|
catch (expErr) {
|
|
309
361
|
await haltMigration(migration, `snapshot failed (${snapErr?.message ?? String(snapErr)}) and the content-only export fallback also failed (${expErr?.message ?? String(expErr)})`, { deps, startedAt, rowsDone: 0, rowsRemaining: initialRemaining, hashEnvelopeMatch: null, state });
|
|
310
|
-
return;
|
|
362
|
+
return false; // halted before any write
|
|
311
363
|
}
|
|
312
364
|
}
|
|
313
365
|
// ── Execution: throttled batches, resumable via per-row markers ──
|
|
@@ -346,7 +398,7 @@ async function runOneMigration(migration, envelope, deps, lock) {
|
|
|
346
398
|
hashEnvelopeMatch: null,
|
|
347
399
|
state,
|
|
348
400
|
});
|
|
349
|
-
return;
|
|
401
|
+
return true; // conservative: a partial batch may have landed before the throw
|
|
350
402
|
}
|
|
351
403
|
// ── Completion gate — strictness per risk class ──
|
|
352
404
|
setMigrationProgress({ id: migration.id, rowsDone, rowsRemaining: 0, state: "completing" });
|
|
@@ -363,7 +415,7 @@ async function runOneMigration(migration, envelope, deps, lock) {
|
|
|
363
415
|
hashEnvelopeMatch: null,
|
|
364
416
|
state,
|
|
365
417
|
});
|
|
366
|
-
return;
|
|
418
|
+
return rowsDone > 0;
|
|
367
419
|
}
|
|
368
420
|
let hashEnvelopeMatch = null;
|
|
369
421
|
let gateOk = finalRemaining === 0;
|
|
@@ -417,7 +469,7 @@ async function runOneMigration(migration, envelope, deps, lock) {
|
|
|
417
469
|
hashEnvelopeMatch,
|
|
418
470
|
state,
|
|
419
471
|
});
|
|
420
|
-
return;
|
|
472
|
+
return rowsDone > 0;
|
|
421
473
|
}
|
|
422
474
|
}
|
|
423
475
|
else if (gateOk && posture.gate === "count+old-row-envelope+new-row-presence") {
|
|
@@ -459,7 +511,7 @@ async function runOneMigration(migration, envelope, deps, lock) {
|
|
|
459
511
|
hashEnvelopeMatch,
|
|
460
512
|
state,
|
|
461
513
|
});
|
|
462
|
-
return;
|
|
514
|
+
return rowsDone > 0;
|
|
463
515
|
}
|
|
464
516
|
// ── Success: ledger → state → prune ──
|
|
465
517
|
const evt = {
|
|
@@ -496,4 +548,5 @@ async function runOneMigration(migration, envelope, deps, lock) {
|
|
|
496
548
|
}
|
|
497
549
|
pruneMigrationSnapshots(deps.snapshotRoot);
|
|
498
550
|
setMigrationProgress({ id: migration.id, rowsDone, rowsRemaining: 0, state: "completed" });
|
|
551
|
+
return rowsDone > 0;
|
|
499
552
|
}
|
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
* encodes the migration id, so "is this row done" is answered entirely by
|
|
28
28
|
* re-reading the row itself (invariant IV: state lives IN the data).
|
|
29
29
|
*/
|
|
30
|
-
import { databases } from "
|
|
30
|
+
import { databases } from "harper";
|
|
31
31
|
export const SYNTHETIC_MIGRATION_ID = "synthetic-ci-schema-stamp";
|
|
32
32
|
export const RESERVED_TEST_AGENT_ID = "__flair_migration_synthetic_test_agent__";
|
|
33
33
|
export const SYNTHETIC_TARGET_MARKER = `${SYNTHETIC_MIGRATION_ID}-done`;
|