@tpsdev-ai/flair 0.28.0 → 0.30.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/README.md +1 -1
  2. package/dist/bridges/builtins/index.js +1 -1
  3. package/dist/bridges/discover.js +1 -1
  4. package/dist/bridges/runtime/load-plugin.js +1 -1
  5. package/dist/bridges/scaffold.js +3 -3
  6. package/dist/bridges/types.js +1 -1
  7. package/dist/cli-shim.cjs +1 -1
  8. package/dist/cli.js +357 -130
  9. package/dist/deploy.js +29 -19
  10. package/dist/fabric-upgrade.js +61 -23
  11. package/dist/fleet-verify.js +3 -3
  12. package/dist/lib/mcp-enable.js +2 -2
  13. package/dist/rem/runner.js +2 -2
  14. package/dist/rem/scheduler.js +226 -17
  15. package/dist/resources/A2AAdapter.js +1 -1
  16. package/dist/resources/Admin.js +1 -1
  17. package/dist/resources/AdminConnectors.js +1 -1
  18. package/dist/resources/AdminDashboard.js +1 -1
  19. package/dist/resources/AdminIdp.js +1 -1
  20. package/dist/resources/AdminInstance.js +1 -1
  21. package/dist/resources/AdminMemory.js +1 -1
  22. package/dist/resources/AdminPrincipals.js +1 -1
  23. package/dist/resources/Agent.js +1 -1
  24. package/dist/resources/AgentCard.js +1 -1
  25. package/dist/resources/AgentSeed.js +1 -1
  26. package/dist/resources/AttentionQuery.js +1 -1
  27. package/dist/resources/Credential.js +1 -1
  28. package/dist/resources/Federation.js +1 -1
  29. package/dist/resources/Instance.js +1 -1
  30. package/dist/resources/Integration.js +1 -1
  31. package/dist/resources/MCPClientMetadata.js +1 -1
  32. package/dist/resources/Memory.js +1 -1
  33. package/dist/resources/MemoryBootstrap.js +1 -1
  34. package/dist/resources/MemoryCandidate.js +168 -0
  35. package/dist/resources/MemoryConsolidate.js +1 -1
  36. package/dist/resources/MemoryDedupStats.js +1 -1
  37. package/dist/resources/MemoryFeed.js +1 -1
  38. package/dist/resources/MemoryGrant.js +1 -1
  39. package/dist/resources/MemoryMaintenance.js +1 -1
  40. package/dist/resources/MemoryReflect.js +3 -3
  41. package/dist/resources/MemoryReindex.js +1 -1
  42. package/dist/resources/MemoryUsage.js +1 -1
  43. package/dist/resources/OAuth.js +1 -1
  44. package/dist/resources/OAuthClient.js +1 -1
  45. package/dist/resources/OrgEvent.js +1 -1
  46. package/dist/resources/OrgEventCatchup.js +1 -1
  47. package/dist/resources/OrgEventMaintenance.js +1 -1
  48. package/dist/resources/PairingToken.js +1 -1
  49. package/dist/resources/Peer.js +1 -1
  50. package/dist/resources/Presence.js +3 -3
  51. package/dist/resources/RecordUsage.js +16 -2
  52. package/dist/resources/Relationship.js +1 -1
  53. package/dist/resources/SemanticSearch.js +1 -1
  54. package/dist/resources/SkillScan.js +1 -1
  55. package/dist/resources/Soul.js +1 -1
  56. package/dist/resources/SoulFeed.js +1 -1
  57. package/dist/resources/WorkspaceLatest.js +1 -1
  58. package/dist/resources/WorkspaceState.js +1 -1
  59. package/dist/resources/XAA.js +1 -1
  60. package/dist/resources/a2a-url.js +1 -1
  61. package/dist/resources/agent-auth.js +1 -1
  62. package/dist/resources/agentcard-fields.js +1 -1
  63. package/dist/resources/auth-middleware.js +1 -1
  64. package/dist/resources/embeddings-boot.js +4 -4
  65. package/dist/resources/embeddings-provider.js +13 -43
  66. package/dist/resources/federation-cleanup.js +5 -5
  67. package/dist/resources/federation-nonce-store.js +5 -5
  68. package/dist/resources/health.js +29 -6
  69. package/dist/resources/instance-identity.js +1 -1
  70. package/dist/resources/mcp-client-metadata-fields.js +1 -1
  71. package/dist/resources/mcp-handler.js +1 -1
  72. package/dist/resources/mcp-oauth.js +2 -2
  73. package/dist/resources/mcp-tools.js +9 -1
  74. package/dist/resources/memory-read-scope.js +1 -1
  75. package/dist/resources/memory-reflect-lib.js +4 -4
  76. package/dist/resources/memory-visibility.js +3 -3
  77. package/dist/resources/migration-boot.js +1 -1
  78. package/dist/resources/migrations/embedding-stamp.js +3 -3
  79. package/dist/resources/migrations/graph-heal.js +2 -2
  80. package/dist/resources/migrations/ledger.js +1 -1
  81. package/dist/resources/migrations/registry.js +14 -5
  82. package/dist/resources/migrations/runner.js +64 -11
  83. package/dist/resources/migrations/synthetic-test-migration.js +1 -1
  84. package/dist/resources/migrations/visibility-backfill.js +232 -0
  85. package/dist/resources/models-dir.js +55 -0
  86. package/dist/resources/record-types.js +30 -2
  87. package/dist/resources/rerank-provider.js +24 -4
  88. package/dist/resources/semantic-retrieval-core.js +1 -1
  89. package/dist/resources/usage-recording.js +73 -7
  90. package/dist/resources/version.js +33 -0
  91. package/dist/src/rem/scheduler.js +449 -0
  92. package/docs/assets/flair-cross-orchestrator.cast +1 -1
  93. package/docs/bridges.md +3 -3
  94. package/docs/deployment.md +1 -0
  95. package/docs/mcp-clients.md +3 -1
  96. package/docs/n8n.md +1 -2
  97. package/docs/notes/rem-ux.md +1 -1
  98. package/docs/rem.md +2 -0
  99. package/docs/supply-chain-policy.md +4 -4
  100. package/docs/upgrade.md +2 -2
  101. package/package.json +4 -4
  102. package/schemas/memory.graphql +27 -12
@@ -36,11 +36,11 @@
36
36
  * what this migration removes: `models.embed` is a process-wide singleton
37
37
  * Harper initializes at boot, off the first-call dynamic-import race.
38
38
  *
39
- * `@harperfast/harper` is dynamic-imported here (deferred to first actual
39
+ * `harper` is dynamic-imported here (deferred to first actual
40
40
  * getEmbedding()/getMode()/getStatus() call), NOT statically imported like
41
41
  * every other resource's `Resource`/`databases`/`server` — those work
42
42
  * unmocked-import-free only because every test that (transitively) imports
43
- * them already mocks `@harperfast/harper` first. Statically importing
43
+ * them already mocks `harper` first. Statically importing
44
44
  * `models` here would make *any* test that imports this file for
45
45
  * `resolveModelsDir()` alone (test/unit/embeddings-models-dir.test.ts, which
46
46
  * has no reason to know about Harper at all) also eagerly load Harper's real
@@ -107,9 +107,6 @@
107
107
  * still a separate, deliberate step from shipping this code — see the PR
108
108
  * description for that distinction.
109
109
  */
110
- import { join } from "node:path";
111
- import { existsSync } from "node:fs";
112
- import { homedir } from "node:os";
113
110
  const VALID_INPUT_TYPES = new Set(["document", "query"]);
114
111
  /**
115
112
  * THE GATE (flair#504 Phase 2 — flipped ON). A module-level constant, not an
@@ -189,7 +186,7 @@ function prefixesEnabled() {
189
186
  * Build the options object passed to `models.embed()`. Pulled out as its own
190
187
  * pure, harper-free function so the value-forwarding (and the reject-a-
191
188
  * wrong-value guard) is unit-testable without touching the deferred
192
- * `@harperfast/harper` import this file's header explains — see
189
+ * `harper` import this file's header explains — see
193
190
  * test/unit/embeddings-provider-input-type.test.ts.
194
191
  *
195
192
  * Gate ON (default): rejects anything other than the literal
@@ -220,7 +217,7 @@ let _modelsApi;
220
217
  /**
221
218
  * Resolve (and cache) Harper's `models` facade via a deferred import — see file header.
222
219
  *
223
- * NOTE for anyone using this as a reference: `models` (the `@harperfast/harper`
220
+ * NOTE for anyone using this as a reference: `models` (the `harper`
224
221
  * package export) and a component's `scope.models` are the SAME boot-time
225
222
  * singleton — Harper's jsLoader hands components `scope.models = <the global
226
223
  * models>` (two accessors, one object; the model registry lives on that singleton).
@@ -232,49 +229,22 @@ let _modelsApi;
232
229
  async function getModelsApi() {
233
230
  if (_modelsApi)
234
231
  return _modelsApi;
235
- const harper = await import("@harperfast/harper");
232
+ const harper = await import("harper");
236
233
  _modelsApi = harper.models;
237
234
  return _modelsApi;
238
235
  }
239
236
  /**
240
237
  * Resolve the directory the embeddings model lives in / downloads into.
241
238
  *
242
- * Resolution order (everything writable, never the read-only package dir):
243
- * 1. FLAIR_MODELS_DIR — explicit operator/docker override.
244
- * 2. <ROOTPATH>/models — Harper's data dir (Flair passes ROOTPATH =
245
- * ~/.flair/data when it spawns Harper). User-
246
- * owned and writable even on sudo-global installs.
247
- * 3. <cwd>/models — ONLY if a model already lives there. Backward
248
- * compat for existing writable installs that
249
- * downloaded into the package dir before this fix;
250
- * never used as a download target on fresh installs.
251
- * 4. ~/.flair/data/models — last-resort default when ROOTPATH is unset.
252
- *
253
- * The chosen dir is always writable, so the embeddings engine can download the
254
- * model on first use without hitting EACCES on a root-owned package dir.
255
- *
256
- * Called by `resources/embeddings-boot.ts` (flair#694) to build the
257
- * `modelsDir` it passes to harper-fabric-embeddings' `register()` — both run
258
- * in the SAME process now (unlike the retired src/cli.ts-side computation,
259
- * which duplicated this exact default because it ran in a separate process
260
- * that set the resulting value via an env var). Still exported and tested
261
- * independently (test/unit/embeddings-models-dir.test.ts) as the single
262
- * documented source of truth for this default.
239
+ * Implementation moved to `resources/models-dir.ts` (flair#815) so the
240
+ * reranker can share the exact same resolution without importing this module
241
+ * (several unit-isolated tests `mock.module()` this file wholesale — see
242
+ * models-dir.ts's header). Re-exported here so existing consumers keep their
243
+ * import site: `resources/embeddings-boot.ts` (flair#694), which builds the
244
+ * `modelsDir` it passes to harper-fabric-embeddings' `register()`, and
245
+ * test/unit/embeddings-models-dir.test.ts, which pins the resolution order.
263
246
  */
264
- export function resolveModelsDir() {
265
- const override = process.env.FLAIR_MODELS_DIR;
266
- if (override)
267
- return override;
268
- const rootPath = process.env.ROOTPATH;
269
- if (rootPath)
270
- return join(rootPath, "models");
271
- // Backward compat: a prior (writable) install may have the model cached in
272
- // the package dir. Reuse it rather than re-downloading — but only if present.
273
- const cwdModels = join(process.cwd(), "models");
274
- if (existsSync(cwdModels))
275
- return cwdModels;
276
- return join(homedir(), ".flair", "data", "models");
277
- }
247
+ export { resolveModelsDir } from "./models-dir.js";
278
248
  // Kern: "Boot probe MUST have a 5-10s timeout ... on timeout set mode='none'
279
249
  // + log a warning. A hung probe ... must never block Harper boot."
280
250
  const PROBE_TIMEOUT_MS = 8_000;
@@ -11,13 +11,13 @@ let cleanupTimer = null;
11
11
  * and performs housekeeping on expired/unconsumed token records.
12
12
  *
13
13
  * In test environments, callers pass mock serverOp/db via `opts` so this
14
- * module never imports @harperfast/harper at the top level (which would
14
+ * module never imports harper at the top level (which would
15
15
  * crash when STORAGE_PATH isn't set).
16
16
  */
17
17
  export async function initFederationCleanup(opts) {
18
18
  const immediate = opts?.immediateTick ?? true;
19
19
  // Resolve server / databases: use caller-supplied mocks when available,
20
- // otherwise lazy-import @harperfast/harper at call time.
20
+ // otherwise lazy-import harper at call time.
21
21
  let svr;
22
22
  let db;
23
23
  if (opts?.serverOp && opts?.db) {
@@ -26,12 +26,12 @@ export async function initFederationCleanup(opts) {
26
26
  }
27
27
  else {
28
28
  try {
29
- const harper = await import("@harperfast/harper");
29
+ const harper = await import("harper");
30
30
  svr = opts?.serverOp ?? harper.server.operation;
31
31
  db = opts?.db ?? harper.databases;
32
32
  }
33
33
  catch (err) {
34
- console.error("[federation-cleanup] failed to load @harperfast/harper:", err?.message ?? err);
34
+ console.error("[federation-cleanup] failed to load harper:", err?.message ?? err);
35
35
  return;
36
36
  }
37
37
  }
@@ -94,7 +94,7 @@ export async function runCleanupTick(opts = {}) {
94
94
  db = opts.db;
95
95
  }
96
96
  else {
97
- const harper = await import("@harperfast/harper");
97
+ const harper = await import("harper");
98
98
  svr = opts.serverOp ?? harper.server.operation;
99
99
  db = opts.db ?? harper.databases;
100
100
  }
@@ -44,7 +44,7 @@ async function resolveDb(opts, cache) {
44
44
  if (opts.db)
45
45
  return opts.db;
46
46
  if (!cache.db) {
47
- cache.db = import("@harperfast/harper").then((h) => h.databases);
47
+ cache.db = import("harper").then((h) => h.databases);
48
48
  }
49
49
  return cache.db;
50
50
  }
@@ -123,7 +123,7 @@ export async function runNonceCleanupTick(opts = {}) {
123
123
  db = opts.db;
124
124
  }
125
125
  else {
126
- const harper = await import("@harperfast/harper");
126
+ const harper = await import("harper");
127
127
  db = harper.databases;
128
128
  }
129
129
  const retentionMs = opts.retentionMs ?? DEFAULT_RETENTION_MS;
@@ -159,7 +159,7 @@ export async function runNonceCleanupTick(opts = {}) {
159
159
  * Initialise the periodic Nonce-table eviction sweep (5-min cadence).
160
160
  *
161
161
  * In test environments, callers pass a mock `db` via `opts` so this module
162
- * never imports @harperfast/harper at the top level (mirrors
162
+ * never imports harper at the top level (mirrors
163
163
  * federation-cleanup.ts's `initFederationCleanup`).
164
164
  */
165
165
  export async function initNonceStoreCleanup(opts) {
@@ -170,11 +170,11 @@ export async function initNonceStoreCleanup(opts) {
170
170
  }
171
171
  else {
172
172
  try {
173
- const harper = await import("@harperfast/harper");
173
+ const harper = await import("harper");
174
174
  db = harper.databases;
175
175
  }
176
176
  catch (err) {
177
- console.error("[federation-nonce-store] failed to load @harperfast/harper:", err?.message ?? err);
177
+ console.error("[federation-nonce-store] failed to load harper:", err?.message ?? err);
178
178
  return;
179
179
  }
180
180
  }
@@ -1,4 +1,4 @@
1
- import { Resource, databases } from "@harperfast/harper";
1
+ import { Resource, databases } from "harper";
2
2
  import { promises as fsp, existsSync, readFileSync } from "node:fs";
3
3
  import { homedir, platform } from "node:os";
4
4
  import { join, dirname } from "node:path";
@@ -452,13 +452,33 @@ export class HealthDetail extends Resource {
452
452
  const lastLightAt = findLast("light");
453
453
  const lastRapidAt = findLast("rapid");
454
454
  const lastRestorativeAt = findLast("restorative");
455
+ // File presence only proves the scheduler templates were written —
456
+ // NOT that launchctl/systemctl actually loaded the job. A
457
+ // `systemctl --user enable --now` that fails (e.g. no session bus)
458
+ // still leaves the timer file on disk, so file-existence alone would
459
+ // report "enabled" for a scheduler that has nothing scheduled
460
+ // (flair#850). Query genuine active state once the file is known to
461
+ // exist; async + short-timeout so a slow/hung service manager can't
462
+ // stall this request.
455
463
  let nightlyEnabled = null;
464
+ let nightlyInstalled = false;
456
465
  const plat = platform();
457
- if (plat === "darwin") {
458
- nightlyEnabled = await exists(join(homedir(), "Library", "LaunchAgents", "dev.flair.rem.nightly.plist"));
459
- }
460
- else if (plat === "linux") {
461
- nightlyEnabled = await exists(join(homedir(), ".config", "systemd", "user", "flair-rem-nightly.timer"));
466
+ if (plat === "darwin" || plat === "linux") {
467
+ nightlyInstalled = plat === "darwin"
468
+ ? await exists(join(homedir(), "Library", "LaunchAgents", "dev.flair.rem.nightly.plist"))
469
+ : await exists(join(homedir(), ".config", "systemd", "user", "flair-rem-nightly.timer"));
470
+ if (nightlyInstalled) {
471
+ try {
472
+ const { queryActiveStateAsync } = await import("../src/rem/scheduler.js");
473
+ nightlyEnabled = await queryActiveStateAsync(plat);
474
+ }
475
+ catch {
476
+ nightlyEnabled = null;
477
+ }
478
+ }
479
+ else {
480
+ nightlyEnabled = false;
481
+ }
462
482
  }
463
483
  const nightlyRecords = await tailJsonl(nightlyLog);
464
484
  const lastNightlyRec = nightlyRecords[nightlyRecords.length - 1];
@@ -496,6 +516,9 @@ export class HealthDetail extends Resource {
496
516
  if (nightlyEnabled && lastNightlyAt && nowMs - new Date(lastNightlyAt).getTime() > 48 * 3600 * 1000) {
497
517
  warnings.push({ level: "warn", message: "nightly REM hasn't run in >48h" });
498
518
  }
519
+ if (nightlyInstalled && nightlyEnabled === false) {
520
+ warnings.push({ level: "warn", message: "REM nightly scheduler files are written but not active — nothing is scheduled (see `flair rem nightly status`)" });
521
+ }
499
522
  }
500
523
  }
501
524
  catch {
@@ -1,4 +1,4 @@
1
- import { databases } from "@harperfast/harper";
1
+ import { databases } from "harper";
2
2
  /**
3
3
  * ─── Local instance identity (federation-edge-hardening slice 1) ────────────
4
4
  *
@@ -39,7 +39,7 @@
39
39
  * `allowedHosts` configured → rejected; leaked private `d` in a JWK →
40
40
  * rejected by the plugin even if our own build-time guard were bypassed).
41
41
  *
42
- * Kept free of any @harperfast/harper import (mirrors agentcard-fields.ts)
42
+ * Kept free of any harper import (mirrors agentcard-fields.ts)
43
43
  * so the document shape is unit-testable without spinning up Harper.
44
44
  */
45
45
  /** The grant this agent's CIMD document targets — formalized by oauth#161, accepted by the published 2.2.0 validator. See module header. */
@@ -23,7 +23,7 @@
23
23
  * Harper HTTP listeners return `{ status, body, headers? }`. MCP messages are
24
24
  * JSON-RPC 2.0, so we serialize the JSON-RPC response object as the body.
25
25
  */
26
- import { databases } from "@harperfast/harper";
26
+ import { databases } from "harper";
27
27
  import { randomBytes } from "node:crypto";
28
28
  import { TOOLS, listToolDefs } from "./mcp-tools.js";
29
29
  // The MCP protocol revision we implement (initialize handshake).
@@ -17,7 +17,7 @@
17
17
  * only meaningful when an operator has set the issuer + enabled the flag (see
18
18
  * docs). The plugin serves DCR / authorize / token / JWKS / discovery.
19
19
  */
20
- import * as harper from "@harperfast/harper";
20
+ import * as harper from "harper";
21
21
  import { mcpOAuthEnabled, mcpAuthConfig } from "./mcp-oauth-flag.js";
22
22
  async function defaultLoadWithMCPAuth() {
23
23
  // Dynamic import so the dep is only required when the surface is enabled.
@@ -53,7 +53,7 @@ export async function registerMcpOAuthRoute(deps = {}) {
53
53
  const handler = deps.mcpHandler ?? (await import("./mcp-handler.js")).mcpHandler;
54
54
  // Read `server` lazily off the namespace (it's a runtime global on the Harper
55
55
  // module, not a static named export) so this module links cleanly even where a
56
- // stub build of @harperfast/harper lacks the export.
56
+ // stub build of harper lacks the export.
57
57
  const srv = deps.server ?? (harper.server);
58
58
  // Primary registration: urlPath subroute → own chain (flair's auth-middleware
59
59
  // does not run here). `getConfig` pins iss/resource to the AS's values so the
@@ -35,6 +35,7 @@
35
35
  * version bump + a new FlairClient method, out of scope for this query-only
36
36
  * slice.
37
37
  */
38
+ import { resolveVersion } from "./version.js";
38
39
  const H = {};
39
40
  const LOADERS = {
40
41
  SemanticSearch: async () => (await import("./SemanticSearch.js")).SemanticSearch,
@@ -248,7 +249,14 @@ async function bootstrap(agent, args) {
248
249
  // ONLY when requested so a plain bootstrap delegates a byte-identical body.
249
250
  if (args?.abstain === true)
250
251
  body.abstain = true;
251
- return unwrap(await h.post(body));
252
+ // flair#831 — attach the running Flair version to the RESPONSE (not the
253
+ // delegated request body) so the calling agent learns the server version
254
+ // on its very first call.
255
+ const result = unwrap(await h.post(body));
256
+ if (result && typeof result === "object" && !Array.isArray(result)) {
257
+ return { ...result, flairVersion: resolveVersion() };
258
+ }
259
+ return result;
252
260
  }
253
261
  async function soulSet(agent, args) {
254
262
  const Cls = await handler("Soul");
@@ -1,4 +1,4 @@
1
- import { databases } from "@harperfast/harper";
1
+ import { databases } from "harper";
2
2
  import { PRIVATE_VISIBILITY, isPrivateVisibility } from "./memory-visibility.js";
3
3
  /**
4
4
  * ─── Centralized Memory read-scoping (the original grant-gated read model → within-org-read-open) ─
@@ -1,6 +1,6 @@
1
1
  // ─── Memory Reflection — pure logic for /ReflectMemories ────────────────────
2
2
  // Pure helpers backing resources/MemoryReflect.ts (FLAIR-NIGHTLY-REM slice 2,
3
- // §3A — see specs/FLAIR-NIGHTLY-REM-SLICE-2-DISTILLATION.md, issue #707).
3
+ // §3A — see issue #707).
4
4
  //
5
5
  // Same split as resources/memory-consolidate-lib.ts: importing MemoryReflect.ts
6
6
  // pulls in the Harper runtime (`databases`/`Resource`/`models`, storage init),
@@ -44,7 +44,7 @@ export const GENERATE_TEMPERATURE = 0.2;
44
44
  //
45
45
  // Passed as `responseFormat: { schema: CANDIDATES_SCHEMA }` to models.generate()
46
46
  // so backends that honor structured output (Ollama, OpenAI — verified against
47
- // the pinned @harperfast/harper 5.1.17's bundled backends) return conformant
47
+ // the pinned harper 5.1.17's bundled backends) return conformant
48
48
  // JSON directly. Not every backend enforces it (Anthropic's Messages API has
49
49
  // no equivalent and Harper documents that it silently ignores the option) —
50
50
  // this module never trusts the backend to have enforced the schema; every
@@ -226,7 +226,7 @@ export function parseAndValidateCandidates(raw, gatheredMemoryIds) {
226
226
  /**
227
227
  * Name Harper's models facade sets on the error it throws when no backend is
228
228
  * registered for the requested logical name (`ModelBackendNotFoundError`,
229
- * resources/models/backendRegistry.ts in @harperfast/harper 5.1.17). That
229
+ * resources/models/backendRegistry.ts in harper 5.1.17). That
230
230
  * class isn't part of the package's public export surface (only its
231
231
  * `Models`/type surface is re-exported from the package root), so detecting
232
232
  * it here is a documented duck-type on `.name` rather than `instanceof` —
@@ -237,7 +237,7 @@ export const MODEL_BACKEND_NOT_FOUND_ERROR_NAME = "ModelBackendNotFoundError";
237
237
  * Calls generate(), validates the result, and on malformed/mismatched output
238
238
  * retries exactly once with an explicit `responseFormat: 'json'` (the
239
239
  * "json-fallback path" — spec §3A items 2 & 3: build-time check confirmed
240
- * `GenerateOpts.responseFormat` supports `{ schema }` in @harperfast/harper
240
+ * `GenerateOpts.responseFormat` supports `{ schema }` in harper
241
241
  * 5.1.17's types, but not every backend enforces it, so the first attempt
242
242
  * requests schema mode and the fallback attempt requests plain json mode).
243
243
  * Both attempts run through the SAME parseAndValidateCandidates — a parse
@@ -9,18 +9,18 @@
9
9
  * - src/cli.ts's runFederationSyncOnce() — the federation-sync PUSH filter
10
10
  * that must not replicate `private` memories to peer instances.
11
11
  *
12
- * Deliberately has ZERO imports — not even "@harperfast/harper". That is
12
+ * Deliberately has ZERO imports — not even "harper". That is
13
13
  * intentional and load-bearing: src/cli.ts is a standalone CLI entrypoint
14
14
  * that runs OUTSIDE any running Harper instance (e.g. `flair federation
15
15
  * sync` invoked from a cron/launchd job). resources/memory-read-scope.ts
16
- * imports `databases` from "@harperfast/harper", and that package's
16
+ * imports `databases` from "harper", and that package's
17
17
  * top-level init eagerly resolves storage paths and THROWS when there is no
18
18
  * live Harper runtime backing it (confirmed empirically — it takes down
19
19
  * even `flair --help`). So src/cli.ts must never import
20
20
  * resources/memory-read-scope.ts (or anything else that drags that
21
21
  * side-effecting import in) directly. This module is the safe seam: a pure
22
22
  * function + constant that both sides can import without dragging in
23
- * "@harperfast/harper".
23
+ * "harper".
24
24
  *
25
25
  * ── The migration invariant (non-negotiable, mirrors memory-read-scope.ts) ──
26
26
  * A record with NO `visibility` field (written before the field existed) is
@@ -26,7 +26,7 @@
26
26
  * the `.catch()` below is pure defense-in-depth so a bug there can never
27
27
  * take down the process either.
28
28
  */
29
- import { databases } from "@harperfast/harper";
29
+ import { databases } from "harper";
30
30
  import { homedir } from "node:os";
31
31
  import { existsSync, readFileSync } from "node:fs";
32
32
  import { join, dirname } from "node:path";
@@ -62,8 +62,8 @@
62
62
  * embeddings engine to settle before running migrations at all.
63
63
  *
64
64
  * flair#807 — completion-gate false-negative on 0.10-era / cross-version
65
- * stores, root-caused against the installed `@harperfast/harper@5.1.22`
66
- * source (node_modules/@harperfast/harper/resources/search.ts): for an
65
+ * stores, root-caused against the installed `harper@5.1.22`
66
+ * source (node_modules/harper/resources/search.ts): for an
67
67
  * INDEXED attribute (`embeddingModel` IS `@indexed` — schemas/memory.graphql),
68
68
  * a non-negated `not_equal`/`ne` leaf condition takes `searchByIndex`'s
69
69
  * `index && !skipIndex` branch, which reconstructs a SYNTHETIC partial record
@@ -125,7 +125,7 @@
125
125
  * resources/migrations/graph-heal.ts). Before enabling any FUTURE bulk
126
126
  * re-embed, ship a structural reindex trigger with it.
127
127
  */
128
- import { databases } from "@harperfast/harper";
128
+ import { databases } from "harper";
129
129
  import { getModelId } from "../embeddings-provider.js";
130
130
  function defaultMemoryTable() {
131
131
  return databases.flair.Memory;
@@ -9,7 +9,7 @@
9
9
  * every pre-this-change install) against the schema and, because
10
10
  * `canonicalizeIndexOptions` does NOT inject defaults (`{type:"HNSW"}` and
11
11
  * `{type:"HNSW",M:16}` are DIFFERENT canonical keys — verified against the
12
- * installed @harperfast/harper source, resources/databases.js:
12
+ * installed harper source, resources/databases.js:
13
13
  * canonicalizeIndexOptions + the `indexOptionsChanged` reset of
14
14
  * `lastIndexedKey` to undefined), clears the graph store and rebuilds it
15
15
  * CLEANLY from the already-correct stored vectors. That rebuild heals the
@@ -57,7 +57,7 @@
57
57
  * correctness — it re-PUTs the same vector through the incremental path and
58
58
  * does not rebuild the graph.
59
59
  */
60
- import { databases } from "@harperfast/harper";
60
+ import { databases } from "harper";
61
61
  import { existsSync, readFileSync } from "node:fs";
62
62
  import { dirname, join } from "node:path";
63
63
  import { fileURLToPath } from "node:url";
@@ -17,7 +17,7 @@
17
17
  * resources/MemoryReindex.ts's `_reindex` path already rely on for
18
18
  * server-internal writes with no HTTP caller behind them.
19
19
  */
20
- import { databases } from "@harperfast/harper";
20
+ import { databases } from "harper";
21
21
  function defaultOrgEventTable() {
22
22
  return databases.flair.OrgEvent;
23
23
  }
@@ -1,11 +1,13 @@
1
1
  /**
2
2
  * registry.ts — the MigrationRegistry: an ordered list of registered
3
- * migrations. embedding-stamp and graph-heal always register; the synthetic
4
- * CI-only variant registers ONLY when shouldRegisterSyntheticMigration() is
5
- * true (see synthetic-test-migration.ts's doc for the exact gating rule).
3
+ * migrations. embedding-stamp, graph-heal, and visibility-backfill always
4
+ * register; the synthetic CI-only variant registers ONLY when
5
+ * shouldRegisterSyntheticMigration() is true (see synthetic-test-
6
+ * migration.ts's doc for the exact gating rule).
6
7
  */
7
8
  import { createEmbeddingStampMigration } from "./embedding-stamp.js";
8
9
  import { createGraphHealMigration } from "./graph-heal.js";
10
+ import { createVisibilityBackfillMigration } from "./visibility-backfill.js";
9
11
  import { createSyntheticTestMigration, shouldRegisterSyntheticMigration } from "./synthetic-test-migration.js";
10
12
  export class MigrationRegistry {
11
13
  migrations = [];
@@ -24,17 +26,24 @@ export class MigrationRegistry {
24
26
  }
25
27
  }
26
28
  /**
27
- * Builds the production registry. Always includes embedding-stamp and
28
- * graph-heal; conditionally includes the CI-only synthetic migration.
29
+ * Builds the production registry. Always includes embedding-stamp,
30
+ * graph-heal, and visibility-backfill; conditionally includes the CI-only
31
+ * synthetic migration.
29
32
  *
30
33
  * Order note: graph-heal registers AFTER embedding-stamp so that if a boot
31
34
  * ever runs both, the (verify-only) recall check ledgers the state AFTER any
32
35
  * re-embed work that cycle already applied — never a stale pre-embed reading.
36
+ * visibility-backfill registers last — it touches an unrelated field
37
+ * (`visibility`, never `embedding`/`embeddingModel`) and has no ordering
38
+ * dependency on either of the other two; appended after them simply as
39
+ * newest-added-last, matching how graph-heal was appended after
40
+ * embedding-stamp.
33
41
  */
34
42
  export function buildRegistry(env = process.env) {
35
43
  const registry = new MigrationRegistry();
36
44
  registry.register(createEmbeddingStampMigration());
37
45
  registry.register(createGraphHealMigration());
46
+ registry.register(createVisibilityBackfillMigration());
38
47
  if (shouldRegisterSyntheticMigration(env)) {
39
48
  registry.register(createSyntheticTestMigration());
40
49
  }
@@ -178,9 +178,11 @@ async function runCycleLocked(deps, lock) {
178
178
  if (candidates.length === 0) {
179
179
  return { ran: false, reason: "nothing pending" };
180
180
  }
181
- // ── ONE shared async pre-hash for the whole cycle (K&S: computed once,
182
- // after ready, before any migration's first write; failure → halt all
183
- // pending candidates rather than guessing which ones are safe). ──
181
+ // ── ONE shared async pre-hash at cycle start (K&S: computed after ready,
182
+ // before any migration's first write; failure → halt all pending
183
+ // candidates rather than guessing which ones are safe). Re-baselined
184
+ // below iff an earlier migration's own writes stale it for a later
185
+ // full-envelope gate — see the staleness comment on the loop. ──
184
186
  setCyclePhase("pre-hash");
185
187
  let envelope;
186
188
  try {
@@ -195,8 +197,49 @@ async function runCycleLocked(deps, lock) {
195
197
  return { ran: false, reason };
196
198
  }
197
199
  setCyclePhase("running");
200
+ // The cycle-start envelope is a valid gate BASELINE only until a migration
201
+ // in this same cycle writes rows: a legitimate, already-gated, ledgered
202
+ // write by an EARLIER migration (e.g. visibility-backfill stamping
203
+ // `visibility` — a SOURCE field) would otherwise read as corruption to a
204
+ // later migration's count+full-envelope gate and spuriously halt it
205
+ // ("completion gate failed: rowsRemaining=0, hash envelope mismatch" — the
206
+ // CI-observed failure on PR #845). That gate exists to catch writes the
207
+ // GATED migration didn't declare, not its predecessors' own completed
208
+ // work. So: track staleness conservatively (ANY rows written by any
209
+ // earlier migration, source-field or not — one cheap recompute beats
210
+ // enumerating which fields each migration touches), and re-baseline with a
211
+ // fresh computeCorpusEnvelope immediately BEFORE a full-envelope consumer
212
+ // starts — still before that migration's own first write, which is exactly
213
+ // the invariant-IV wording ("computed before first write and after
214
+ // completion; must match"). Cycles with no full-envelope consumer after a
215
+ // writing migration (every production cycle today — no schema-additive
216
+ // migration is registered outside CI) never pay for a second corpus walk.
217
+ let envelopeStale = false;
198
218
  for (const migration of candidates) {
199
- await runOneMigration(migration, envelope, deps, lock);
219
+ if (envelopeStale && postureFor(migration.riskClass).gate === "count+full-envelope") {
220
+ try {
221
+ envelope = await computeCorpusEnvelope(deps.getTable, deps.now);
222
+ envelopeStale = false;
223
+ }
224
+ catch (err) {
225
+ // Same failure class as the cycle-start pre-hash, but scoped: only
226
+ // THIS migration consumes the baseline, so only it halts (the
227
+ // cycle-start variant halts every candidate because nothing has run
228
+ // yet and a failing corpus walk taints them all equally). Halted ≠
229
+ // bricked: it is retried on the next boot's cycle.
230
+ setMigrationProgress({
231
+ id: migration.id,
232
+ rowsDone: 0,
233
+ rowsRemaining: 0,
234
+ state: "halted",
235
+ reason: `pre-flight integrity check failed: ${err?.message ?? String(err)}`,
236
+ });
237
+ continue;
238
+ }
239
+ }
240
+ const wroteRows = await runOneMigration(migration, envelope, deps, lock);
241
+ if (wroteRows)
242
+ envelopeStale = true;
200
243
  }
201
244
  setCyclePhase("done");
202
245
  return { ran: true };
@@ -243,6 +286,15 @@ async function haltMigration(migration, reason, ctx) {
243
286
  /* best-effort — the in-memory progress + ledger event already recorded the halt */
244
287
  }
245
288
  }
289
+ /**
290
+ * Returns whether this migration MAY have written any rows — the cycle
291
+ * loop's envelope-staleness signal (see runCycleLocked). Deliberately
292
+ * conservative: `true` whenever rows were processed OR run() threw
293
+ * mid-batch (a partial batch may have landed before the throw); a false
294
+ * `true` merely costs one extra corpus walk before the next full-envelope
295
+ * gate, while a false `false` would let that gate compare against a stale
296
+ * baseline and spuriously halt.
297
+ */
246
298
  async function runOneMigration(migration, envelope, deps, lock) {
247
299
  const startedAt = deps.now().toISOString();
248
300
  const state = readMigrationState(deps.statePath);
@@ -262,7 +314,7 @@ async function runOneMigration(migration, envelope, deps, lock) {
262
314
  hashEnvelopeMatch: null,
263
315
  state,
264
316
  });
265
- return;
317
+ return false; // halted before any write
266
318
  }
267
319
  const estSnapshot = estimateSnapshotBytes(posture.snapshotScope, initialRemaining);
268
320
  const estWorkingSet = estimateWorkingSetBytes(migration.riskClass, initialRemaining);
@@ -282,7 +334,7 @@ async function runOneMigration(migration, envelope, deps, lock) {
282
334
  hashEnvelopeMatch: null,
283
335
  state,
284
336
  });
285
- return;
337
+ return false; // halted before any write
286
338
  }
287
339
  // ── Ladder step 3 (+ step 4 fallback): risk-scoped snapshot, or a
288
340
  // content-only export if the snapshot mechanism itself fails. ──
@@ -307,7 +359,7 @@ async function runOneMigration(migration, envelope, deps, lock) {
307
359
  }
308
360
  catch (expErr) {
309
361
  await haltMigration(migration, `snapshot failed (${snapErr?.message ?? String(snapErr)}) and the content-only export fallback also failed (${expErr?.message ?? String(expErr)})`, { deps, startedAt, rowsDone: 0, rowsRemaining: initialRemaining, hashEnvelopeMatch: null, state });
310
- return;
362
+ return false; // halted before any write
311
363
  }
312
364
  }
313
365
  // ── Execution: throttled batches, resumable via per-row markers ──
@@ -346,7 +398,7 @@ async function runOneMigration(migration, envelope, deps, lock) {
346
398
  hashEnvelopeMatch: null,
347
399
  state,
348
400
  });
349
- return;
401
+ return true; // conservative: a partial batch may have landed before the throw
350
402
  }
351
403
  // ── Completion gate — strictness per risk class ──
352
404
  setMigrationProgress({ id: migration.id, rowsDone, rowsRemaining: 0, state: "completing" });
@@ -363,7 +415,7 @@ async function runOneMigration(migration, envelope, deps, lock) {
363
415
  hashEnvelopeMatch: null,
364
416
  state,
365
417
  });
366
- return;
418
+ return rowsDone > 0;
367
419
  }
368
420
  let hashEnvelopeMatch = null;
369
421
  let gateOk = finalRemaining === 0;
@@ -417,7 +469,7 @@ async function runOneMigration(migration, envelope, deps, lock) {
417
469
  hashEnvelopeMatch,
418
470
  state,
419
471
  });
420
- return;
472
+ return rowsDone > 0;
421
473
  }
422
474
  }
423
475
  else if (gateOk && posture.gate === "count+old-row-envelope+new-row-presence") {
@@ -459,7 +511,7 @@ async function runOneMigration(migration, envelope, deps, lock) {
459
511
  hashEnvelopeMatch,
460
512
  state,
461
513
  });
462
- return;
514
+ return rowsDone > 0;
463
515
  }
464
516
  // ── Success: ledger → state → prune ──
465
517
  const evt = {
@@ -496,4 +548,5 @@ async function runOneMigration(migration, envelope, deps, lock) {
496
548
  }
497
549
  pruneMigrationSnapshots(deps.snapshotRoot);
498
550
  setMigrationProgress({ id: migration.id, rowsDone, rowsRemaining: 0, state: "completed" });
551
+ return rowsDone > 0;
499
552
  }
@@ -27,7 +27,7 @@
27
27
  * encodes the migration id, so "is this row done" is answered entirely by
28
28
  * re-reading the row itself (invariant IV: state lives IN the data).
29
29
  */
30
- import { databases } from "@harperfast/harper";
30
+ import { databases } from "harper";
31
31
  export const SYNTHETIC_MIGRATION_ID = "synthetic-ci-schema-stamp";
32
32
  export const RESERVED_TEST_AGENT_ID = "__flair_migration_synthetic_test_agent__";
33
33
  export const SYNTHETIC_TARGET_MARKER = `${SYNTHETIC_MIGRATION_ID}-done`;