@tpsdev-ai/flair 0.20.1 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/README.md +39 -11
  2. package/SECURITY.md +52 -20
  3. package/config.yaml +32 -5
  4. package/dist/cli.js +2235 -260
  5. package/dist/deploy.js +96 -15
  6. package/dist/doctor-client.js +312 -0
  7. package/dist/fleet-presence.js +98 -0
  8. package/dist/fleet-verify.js +291 -0
  9. package/dist/install/clients.js +18 -0
  10. package/dist/mcp-client-assertion.js +364 -0
  11. package/dist/probe.js +97 -0
  12. package/dist/rem/restore.js +1 -1
  13. package/dist/rem/runner.js +82 -12
  14. package/dist/resources/Admin.js +1 -1
  15. package/dist/resources/AdminConnectors.js +1 -1
  16. package/dist/resources/AdminDashboard.js +1 -1
  17. package/dist/resources/AdminIdp.js +1 -1
  18. package/dist/resources/AdminInstance.js +1 -1
  19. package/dist/resources/AdminMemory.js +2 -2
  20. package/dist/resources/AdminPrincipals.js +1 -1
  21. package/dist/resources/Agent.js +14 -0
  22. package/dist/resources/AgentCard.js +1 -1
  23. package/dist/resources/AgentSeed.js +1 -1
  24. package/dist/resources/AttentionQuery.js +356 -0
  25. package/dist/resources/Credential.js +11 -1
  26. package/dist/resources/Federation.js +121 -2
  27. package/dist/resources/Integration.js +1 -1
  28. package/dist/resources/MCPClientMetadata.js +88 -0
  29. package/dist/resources/Memory.js +127 -22
  30. package/dist/resources/MemoryBootstrap.js +441 -85
  31. package/dist/resources/MemoryGrant.js +1 -1
  32. package/dist/resources/MemoryReflect.js +105 -49
  33. package/dist/resources/MemoryUsage.js +104 -0
  34. package/dist/resources/OAuth.js +69 -5
  35. package/dist/resources/OrgEvent.js +11 -0
  36. package/dist/resources/OrgEventCatchup.js +1 -1
  37. package/dist/resources/Presence.js +268 -17
  38. package/dist/resources/RecordUsage.js +230 -0
  39. package/dist/resources/Relationship.js +108 -26
  40. package/dist/resources/SemanticSearch.js +88 -320
  41. package/dist/resources/SkillScan.js +15 -0
  42. package/dist/resources/Soul.js +14 -0
  43. package/dist/resources/WorkspaceLatest.js +1 -1
  44. package/dist/resources/WorkspaceState.js +12 -1
  45. package/dist/resources/agent-auth.js +61 -3
  46. package/dist/resources/agentcard-fields.js +2 -2
  47. package/dist/resources/auth-middleware.js +42 -14
  48. package/dist/resources/bm25-filter.js +1 -1
  49. package/dist/resources/bm25.js +13 -7
  50. package/dist/resources/collision-lib.js +157 -0
  51. package/dist/resources/dedup.js +2 -2
  52. package/dist/resources/ed25519-auth.js +2 -2
  53. package/dist/resources/embeddings-boot.js +149 -0
  54. package/dist/resources/embeddings-provider.js +364 -106
  55. package/dist/resources/entity-vocab.js +139 -0
  56. package/dist/resources/federation-nonce-store.js +195 -0
  57. package/dist/resources/health.js +97 -6
  58. package/dist/resources/instance-identity.js +53 -0
  59. package/dist/resources/mcp-client-metadata-fields.js +112 -0
  60. package/dist/resources/mcp-handler.js +1 -1
  61. package/dist/resources/mcp-tools.js +88 -10
  62. package/dist/resources/memory-bootstrap-lib.js +1 -1
  63. package/dist/resources/memory-read-scope.js +58 -71
  64. package/dist/resources/memory-reflect-lib.js +289 -0
  65. package/dist/resources/memory-visibility.js +37 -0
  66. package/dist/resources/migration-boot.js +143 -0
  67. package/dist/resources/migrations/dir-safety.js +71 -0
  68. package/dist/resources/migrations/embedding-stamp.js +178 -0
  69. package/dist/resources/migrations/envelope.js +43 -0
  70. package/dist/resources/migrations/export.js +38 -0
  71. package/dist/resources/migrations/ledger.js +55 -0
  72. package/dist/resources/migrations/lock.js +154 -0
  73. package/dist/resources/migrations/progress.js +31 -0
  74. package/dist/resources/migrations/registry.js +36 -0
  75. package/dist/resources/migrations/risk-policy.js +23 -0
  76. package/dist/resources/migrations/runner.js +456 -0
  77. package/dist/resources/migrations/snapshot.js +127 -0
  78. package/dist/resources/migrations/source-fields.js +93 -0
  79. package/dist/resources/migrations/space.js +73 -0
  80. package/dist/resources/migrations/state.js +64 -0
  81. package/dist/resources/migrations/status.js +39 -0
  82. package/dist/resources/migrations/synthetic-test-migration.js +93 -0
  83. package/dist/resources/migrations/types.js +9 -0
  84. package/dist/resources/presence-internal.js +29 -0
  85. package/dist/resources/provenance.js +50 -0
  86. package/dist/resources/rate-limiter.js +8 -1
  87. package/dist/resources/scoring.js +124 -5
  88. package/dist/resources/semantic-retrieval-core.js +317 -0
  89. package/dist/version-check.js +167 -0
  90. package/dist/version-handshake.js +122 -0
  91. package/package.json +4 -5
  92. package/schemas/agent.graphql +7 -0
  93. package/schemas/event.graphql +5 -0
  94. package/schemas/federation.graphql +12 -0
  95. package/schemas/memory.graphql +82 -0
  96. package/schemas/schema.graphql +5 -43
  97. package/schemas/workspace.graphql +3 -0
  98. package/dist/resources/IngestEvents.js +0 -162
  99. package/dist/resources/IssueTokens.js +0 -19
  100. package/dist/resources/ObsAgentSnapshot.js +0 -13
  101. package/dist/resources/ObsEventFeed.js +0 -13
  102. package/dist/resources/ObsOffice.js +0 -19
  103. package/dist/resources/ObservationCenter.js +0 -23
  104. package/ui/observation-center.html +0 -385
@@ -0,0 +1,143 @@
1
+ /**
2
+ * migration-boot.ts — the boot-keyed trigger (flair#695
3
+ * §A, Kern verdict): "Envelope ASYNC after ready — boot serves immediately
4
+ * on the old shape; pre-hash runs async; migration deferred until it
5
+ * completes; health shows 'pre-flight integrity check in progress'."
6
+ *
7
+ * This is a plain (non-Resource) module — same shape as
8
+ * resources/embeddings-provider.ts / resources/table-helpers.ts / etc. — so
9
+ * Harper's `jsResource: files: dist/resources/*.js` loader (config.yaml)
10
+ * imports it at boot like every other flat file under resources/, running
11
+ * its top-level side effect exactly once per process. It exports no
12
+ * Resource subclass — there is no HTTP endpoint here, only the trigger.
13
+ *
14
+ * Timing: `scheduleMigrationBoot()` defers the actual cycle via
15
+ * `setImmediate`, which runs after the current synchronous phase (module
16
+ * loading / resource registration) yields to the event loop — in practice,
17
+ * after Harper's HTTP listener is already accepting connections, so the
18
+ * server is serving on the OLD shape before any migration write happens
19
+ * (the #687 boot-win property this preserves). As an additional guard
20
+ * against Harper-internal load-ordering this file has no visibility into,
21
+ * the deferred callback ALSO polls for `databases.flair.Memory`/`Relationship`
22
+ * actually being live table accessors before invoking the runner — cheap,
23
+ * bounded, and self-healing if the very first check is too early.
24
+ *
25
+ * `runMigrationCycle` itself never throws (see runner.ts's module doc) —
26
+ * the `.catch()` below is pure defense-in-depth so a bug there can never
27
+ * take down the process either.
28
+ */
29
+ import { databases } from "@harperfast/harper";
30
+ import { homedir } from "node:os";
31
+ import { existsSync, readFileSync } from "node:fs";
32
+ import { join, dirname } from "node:path";
33
+ import { fileURLToPath } from "node:url";
34
+ import { buildRegistry } from "./migrations/registry.js";
35
+ import { runMigrationCycle } from "./migrations/runner.js";
36
+ import { seedIdleProgress } from "./migrations/progress.js";
37
+ import { getMode } from "./embeddings-provider.js";
38
+ /** Same dataDir resolution as resources/health.ts's disk section. */
39
+ export function resolveMigrationDataDir() {
40
+ return process.env.HDB_ROOT ?? join(homedir(), ".flair", "data");
41
+ }
42
+ /** Same "resolve the running package's own version" idiom as resources/health.ts. */
43
+ export function resolveRunningVersion() {
44
+ try {
45
+ const here = dirname(fileURLToPath(import.meta.url));
46
+ const candidates = [join(here, "..", "..", "package.json"), join(here, "..", "package.json")];
47
+ for (const p of candidates) {
48
+ if (existsSync(p)) {
49
+ const pkg = JSON.parse(readFileSync(p, "utf-8"));
50
+ if (pkg.version)
51
+ return pkg.version;
52
+ }
53
+ }
54
+ }
55
+ catch {
56
+ /* fall through */
57
+ }
58
+ return process.env.npm_package_version ?? "dev";
59
+ }
60
+ function getTable(table) {
61
+ return databases.flair[table];
62
+ }
63
+ async function waitForTablesReady(maxWaitMs = 30_000, intervalMs = 200) {
64
+ const deadline = Date.now() + maxWaitMs;
65
+ for (;;) {
66
+ try {
67
+ const flair = databases?.flair;
68
+ const mem = flair?.Memory;
69
+ const rel = flair?.Relationship;
70
+ if (typeof mem?.search === "function" && typeof rel?.search === "function")
71
+ return true;
72
+ }
73
+ catch {
74
+ /* keep polling */
75
+ }
76
+ if (Date.now() >= deadline)
77
+ return false;
78
+ await new Promise((r) => setTimeout(r, intervalMs));
79
+ }
80
+ }
81
+ /**
82
+ * Gives the embeddings engine a bounded window to finish its own boot probe
83
+ * (resources/embeddings-provider.ts: PROBE_TIMEOUT_MS = 8s) before running
84
+ * migrations. Root-cause fix for a real race found while building the
85
+ * embedding-stamp integration test: this trigger fires very early (right
86
+ * after the Memory/Relationship tables exist), which can beat the
87
+ * embeddings engine's own async model-load — a Memory.put() regen attempted
88
+ * during that window silently fails (getEmbedding() catches and returns
89
+ * null), leaving a row's embeddingModel null instead of freshly stamped.
90
+ * `getMode()==="local"` breaks out early on the common case (embeddings
91
+ * already warm); otherwise this waits up to ~8.5s (a hair over the probe's
92
+ * own timeout, so whatever getMode() reports by then is genuinely settled,
93
+ * not just "haven't checked yet") and proceeds regardless — a migration
94
+ * must never block boot indefinitely on this, and embedding-stamp's own
95
+ * pending-condition (OR of not_equal + equals-null — see that file) still
96
+ * self-heals a stray null on the NEXT boot even if this window isn't enough.
97
+ */
98
+ async function waitForEmbeddingsSettled(maxWaitMs = 8_500, intervalMs = 150) {
99
+ const deadline = Date.now() + maxWaitMs;
100
+ while (Date.now() < deadline) {
101
+ if (getMode() === "local")
102
+ return;
103
+ await new Promise((r) => setTimeout(r, intervalMs));
104
+ }
105
+ }
106
+ let scheduled = false;
107
+ export function scheduleMigrationBoot() {
108
+ if (scheduled)
109
+ return;
110
+ scheduled = true;
111
+ const registry = buildRegistry();
112
+ seedIdleProgress(registry.list().map((m) => m.id));
113
+ setImmediate(() => {
114
+ void (async () => {
115
+ const ready = await waitForTablesReady();
116
+ if (!ready) {
117
+ console.error("[flair-migrations] Memory/Relationship tables never became ready — skipping this boot's migration cycle (will retry next boot)");
118
+ return;
119
+ }
120
+ await waitForEmbeddingsSettled();
121
+ try {
122
+ await runMigrationCycle({
123
+ registry,
124
+ getTable,
125
+ dataDir: resolveMigrationDataDir(),
126
+ runningVersion: resolveRunningVersion(),
127
+ });
128
+ }
129
+ catch (err) {
130
+ // Defense-in-depth only — runMigrationCycle is documented to never
131
+ // throw. A boot-path exception must never surface here regardless.
132
+ console.error(`[flair-migrations] unexpected error from runMigrationCycle: ${err?.message ?? String(err)}`);
133
+ }
134
+ })();
135
+ });
136
+ }
137
+ // Test-only reset so a unit/integration test can re-trigger scheduling
138
+ // within the same process (never used in production — a real process only
139
+ // ever boots once).
140
+ export function _resetMigrationBootForTests() {
141
+ scheduled = false;
142
+ }
143
+ scheduleMigrationBoot();
@@ -0,0 +1,71 @@
1
+ /**
2
+ * dir-safety.ts — 0700 directory enforcement for migration snapshot/export
3
+ * dirs (Sherlock verdict: "0700 via stat-verify after creation (umask can
4
+ * override mkdir mode) and before writes; refuse with the path + actual
5
+ * perms named in the error." — "the safety mechanism must not become the
6
+ * exfil surface... export dir = highest-value target on the system.")
7
+ *
8
+ * `ensureSecureDir` creates-or-verifies a directory is 0700: it mkdirs with
9
+ * mode 0700, then explicitly `chmodSync`s to 0700 (covers a permissive
10
+ * umask silently widening mkdir's mode argument), then stat-verifies the
11
+ * result. If the directory STILL isn't 0700 after the explicit chmod
12
+ * (unusual filesystem/ACL), it refuses loudly rather than silently
13
+ * proceeding into a world-readable directory — naming the exact path and
14
+ * actual octal perms in the thrown error, per Sherlock's wording.
15
+ *
16
+ * `verifySecureDir` is exported separately so callers can (and must)
17
+ * re-verify immediately before EACH write inside the directory, not only at
18
+ * creation time (Kern: "Perms re-verified at write time, not only
19
+ * creation.") — a directory can be re-permissioned by something else on the
20
+ * host between creation and a later write in a long-running batch loop.
21
+ */
22
+ import { existsSync, mkdirSync, chmodSync, statSync, writeFileSync } from "node:fs";
23
+ export class UnsafeDirectoryError extends Error {
24
+ path;
25
+ actualMode;
26
+ constructor(path, actualMode) {
27
+ super(`refusing to use ${path}: expected 0700, found ${actualMode.toString(8).padStart(4, "0")} ` +
28
+ `(group/other-accessible) — fix permissions (chmod 700 ${path}) or point the migration ` +
29
+ `at a different directory`);
30
+ this.path = path;
31
+ this.actualMode = actualMode;
32
+ this.name = "UnsafeDirectoryError";
33
+ }
34
+ }
35
+ /** Throws UnsafeDirectoryError if `dir`'s mode has any group/other bits set. */
36
+ export function verifySecureDir(dir) {
37
+ const st = statSync(dir);
38
+ const mode = st.mode & 0o777;
39
+ if ((mode & 0o077) !== 0) {
40
+ throw new UnsafeDirectoryError(dir, mode);
41
+ }
42
+ }
43
+ /**
44
+ * Ensures `dir` exists and is 0700. Idempotent — safe to call before every
45
+ * write. Never silently tolerates a world/group-readable result: attempts
46
+ * one explicit chmod(0700) remediation (covers the umask-widened-mkdir
47
+ * case), then re-verifies and throws if it's still not compliant.
48
+ */
49
+ export function ensureSecureDir(dir) {
50
+ if (!existsSync(dir)) {
51
+ mkdirSync(dir, { recursive: true, mode: 0o700 });
52
+ }
53
+ try {
54
+ verifySecureDir(dir);
55
+ }
56
+ catch (err) {
57
+ if (!(err instanceof UnsafeDirectoryError))
58
+ throw err;
59
+ chmodSync(dir, 0o700);
60
+ verifySecureDir(dir); // throws again (loudly) if the chmod didn't stick
61
+ }
62
+ }
63
+ /**
64
+ * Writes `contents` to `path` at 0600, re-verifying the CONTAINING
65
+ * directory is still 0700 immediately before the write (per Kern's
66
+ * write-time re-verification requirement above).
67
+ */
68
+ export function writeSecureFile(path, contents, dir) {
69
+ verifySecureDir(dir);
70
+ writeFileSync(path, contents, { mode: 0o600 });
71
+ }
@@ -0,0 +1,178 @@
1
+ /**
2
+ * embedding-stamp.ts — the first registered migration (per the spec):
3
+ * re-embeds every Memory row whose `embeddingModel` doesn't match
4
+ * `getModelId()`. riskClass 'derived-only' — embeddings are recomputable
5
+ * from `content` (SOURCE_FIELDS), never the only copy of anything
6
+ * (invariant I), so this is the cheapest posture: metadata-only snapshot,
7
+ * no content-hash gate (row-count + stamp convergence only).
8
+ *
9
+ * THE GATE (`EMBEDDING_PREFIXES_ENABLED` in resources/embeddings-provider.ts)
10
+ * is now ON (flair#504, flipped and re-baselined through the ratchet gate),
11
+ * so `getModelId()` returns `<base>+searchprefix` — every row written before
12
+ * this flip was stamped with the bare base id, so it now reads as stale and
13
+ * gets picked up by this migration on the next boot that reaches it. This is
14
+ * this migration's first real payload: before the flip, `getModelId()`
15
+ * returned the bare base id unconditionally, so this migration — live and
16
+ * always-registered from the day it shipped — had nothing to detect. That
17
+ * was intentional groundwork, not dead code: proving the detect/re-embed
18
+ * mechanism end-to-end (this file, `test/integration/
19
+ * migrations-embedding-stamp-e2e.test.ts`) before it ever had real work to
20
+ * do was the point.
21
+ *
22
+ * Reuses Memory's OWN regen branch — never duplicates embedding logic —
23
+ * via the SAME mechanism `flair reembed` (src/cli.ts) already uses in
24
+ * production: a genuine `PUT /Memory/:id` HTTP request (admin-authenticated
25
+ * loopback), not an in-process call on `databases.flair.Memory` directly.
26
+ *
27
+ * THIS IS LOAD-BEARING, confirmed empirically while building
28
+ * test/integration/migrations-embedding-stamp-e2e.test.ts against real
29
+ * Harper: `databases.flair.Memory` (the property Harper exposes to OTHER
30
+ * modules, imported the same way resources/health.ts / MemoryReindex.ts /
31
+ * etc. already do for READS) resolves to the RAW underlying table, not the
32
+ * resources/Memory.ts SUBCLASS that carries the actual regen/dedup/auth
33
+ * logic — that subclass is only reachable via Harper's own REST dispatch
34
+ * for a genuine `/Memory` request. Calling `.put()` on the raw reference
35
+ * writes the record fine (confirmed: fields land exactly as passed) but
36
+ * silently skips every override in resources/Memory.ts, INCLUDING the
37
+ * `if (content.content && !content.embedding) { regenerate } ` branch this
38
+ * migration exists to trigger. Reads (`.search()`/`.get()`, used below for
39
+ * detect/countPending/candidate-selection) are unaffected by this — only
40
+ * `.put()` needs the real dispatch, hence the loopback HTTP call for
41
+ * exactly that one step.
42
+ *
43
+ * Query correctness (also verified against real Harper): Harper's
44
+ * `not_equal` comparator only matches rows where the attribute holds an
45
+ * EXPLICIT value (including explicit `null`) — a row where the attribute
46
+ * was NEVER SET AT ALL (`getIndexedValues()` returns `undefined` for a
47
+ * truly-absent property, never indexed regardless of `indexNulls`) is
48
+ * invisible to ANY condition-based query, not just `not_equal`. Clearing to
49
+ * `null` (never `undefined`) on write is therefore load-bearing: if the
50
+ * regen HTTP call fails (engine not yet warmed up, transient failure,
51
+ * admin credential unavailable this cycle), the row must land back in an
52
+ * EXPLICIT-null state — queryable and retried on the next boot — never a
53
+ * truly-absent one that would be permanently invisible to this migration
54
+ * again. The pending condition is an OR of `not_equal <current>` (catches a
55
+ * stale non-null model string) and `equals null` (catches that
56
+ * explicit-null state) — together they catch every state this migration's
57
+ * OWN writes can ever produce. A row whose `embeddingModel` was NEVER
58
+ * touched by anything (truly absent from its very first write — only
59
+ * possible if the embeddings engine was down for that entire write) is a
60
+ * known, narrow gap this bounded query cannot see;
61
+ * resources/migration-boot.ts mitigates the common case by waiting for the
62
+ * embeddings engine to settle before running migrations at all.
63
+ */
64
+ import { databases } from "@harperfast/harper";
65
+ import { getModelId } from "../embeddings-provider.js";
66
+ function defaultMemoryTable() {
67
+ return databases.flair.Memory;
68
+ }
69
+ export const EMBEDDING_STAMP_ID = "embedding-stamp";
70
+ const REGEN_HTTP_TIMEOUT_MS = 20_000; // a real embedding compute can be slow on constrained hardware
71
+ /** Same admin-password resolution as resources/auth-middleware.ts's getAdminPass(). */
72
+ function resolveAdminAuthHeader() {
73
+ const pass = process.env.HDB_ADMIN_PASSWORD ?? process.env.FLAIR_ADMIN_PASSWORD;
74
+ if (!pass)
75
+ return null;
76
+ return "Basic " + Buffer.from(`admin:${pass}`).toString("base64");
77
+ }
78
+ /** Same HTTP_PORT env resolution src/cli.ts sets on every Harper spawn (see that file's grep for HTTP_PORT). */
79
+ function resolveSelfBaseUrl() {
80
+ const port = process.env.HTTP_PORT ?? "9926";
81
+ return `http://127.0.0.1:${port}`;
82
+ }
83
+ /**
84
+ * Triggers Memory.put()'s regen branch via a genuine loopback HTTP PUT —
85
+ * the ONLY reliable way to reach resources/Memory.ts's subclass logic (see
86
+ * module doc). Returns true iff the request succeeded (2xx); NEVER throws
87
+ * — a failure just leaves the row in its current (queryable, explicit-null
88
+ * or stale-string) state for the next attempt.
89
+ */
90
+ async function regenViaHttpPut(id, existing, fetchImpl) {
91
+ const authHeader = resolveAdminAuthHeader();
92
+ if (!authHeader)
93
+ return false; // no admin credential available this cycle — retried next boot
94
+ try {
95
+ const res = await fetchImpl(`${resolveSelfBaseUrl()}/Memory/${encodeURIComponent(id)}`, {
96
+ method: "PUT",
97
+ headers: { "Content-Type": "application/json", Authorization: authHeader },
98
+ body: JSON.stringify({ ...existing, embedding: null, embeddingModel: null }),
99
+ signal: AbortSignal.timeout(REGEN_HTTP_TIMEOUT_MS),
100
+ });
101
+ return res.ok;
102
+ }
103
+ catch {
104
+ return false;
105
+ }
106
+ }
107
+ /**
108
+ * `getTable`/`getCurrentModelId`/`regen` are injectable so unit tests can
109
+ * exercise this migration's full detect/countPending/run logic against an
110
+ * in-memory fake table and a fake regen function (no real Harper, no real
111
+ * HTTP needed) — matching the mocking technique used throughout
112
+ * test/unit/*.ts (e.g. test/unit/instance-identity.test.ts). The real
113
+ * loopback-HTTP regen path is exercised for real in
114
+ * test/integration/migrations-embedding-stamp-e2e.test.ts.
115
+ */
116
+ export function createEmbeddingStampMigration(getTable = defaultMemoryTable, getCurrentModelId = getModelId, regen = (id, existing) => regenViaHttpPut(id, existing, fetch)) {
117
+ function staleCondition() {
118
+ // OR-combined: `not_equal <current>` catches a stale non-null model
119
+ // string; `equals null` catches the explicit-null state this
120
+ // migration's own writes leave behind on a failed regen (see the
121
+ // module doc above — Harper's index never sees a TRULY ABSENT
122
+ // property, only an explicit null).
123
+ return [
124
+ {
125
+ operator: "or",
126
+ conditions: [
127
+ { attribute: "embeddingModel", comparator: "not_equal", value: getCurrentModelId() },
128
+ { attribute: "embeddingModel", comparator: "equals", value: null },
129
+ ],
130
+ },
131
+ ];
132
+ }
133
+ return {
134
+ id: EMBEDDING_STAMP_ID,
135
+ riskClass: "derived-only",
136
+ affectsTables: ["Memory"],
137
+ async detect() {
138
+ const table = getTable();
139
+ for await (const _row of table.search({ conditions: staleCondition(), limit: 1 })) {
140
+ return true;
141
+ }
142
+ return false;
143
+ },
144
+ async countPending() {
145
+ const table = getTable();
146
+ let n = 0;
147
+ for await (const _row of table.search({ conditions: staleCondition() }))
148
+ n++;
149
+ return n;
150
+ },
151
+ async run(batchSize) {
152
+ const table = getTable();
153
+ const current = getCurrentModelId();
154
+ const candidates = [];
155
+ for await (const row of table.search({ conditions: staleCondition(), limit: batchSize })) {
156
+ candidates.push(row);
157
+ }
158
+ const touchedIds = [];
159
+ for (const row of candidates) {
160
+ const id = String(row.id ?? "");
161
+ if (!id)
162
+ continue;
163
+ const existing = await table.get(id);
164
+ if (!existing)
165
+ continue; // deleted since the search above — nothing to fix
166
+ if (existing.embeddingModel === current)
167
+ continue; // already stamped by a concurrent runner — idempotent skip
168
+ const ok = await regen(id, existing);
169
+ if (ok)
170
+ touchedIds.push(id);
171
+ // A failed regen leaves the row untouched (still matching
172
+ // staleCondition — retried next batch/boot), never partially
173
+ // written or marked done.
174
+ }
175
+ return { processed: touchedIds.length, touchedIds };
176
+ },
177
+ };
178
+ }
@@ -0,0 +1,43 @@
1
+ /**
2
+ * envelope.ts — the corpus-wide integrity envelope: "full-corpus
3
+ * hash-of-source-field-hashes computed before first write and after
4
+ * completion; must match" (flair#695 invariant IV /
5
+ * Kern verdict), computed ONCE per boot cycle (K&S: "Envelope ASYNC after
6
+ * ready — boot serves immediately on the old shape; pre-hash runs async;
7
+ * migration deferred until it completes").
8
+ *
9
+ * `computeCorpusEnvelope` walks BOTH SOURCE_FIELDS tables (Memory,
10
+ * Relationship) once and returns both the single aggregate `corpusHash`
11
+ * (what schema-additive's full-envelope gate compares pre/post) AND the
12
+ * full `perRowHash` breakdown (`${table}:${id}` -> hash) — the latter isn't
13
+ * used by this module directly, but keeping the breakdown around lets a
14
+ * caller answer "did THIS specific row's source fields change" without a
15
+ * second full-corpus pass, if ever needed for diagnostics.
16
+ *
17
+ * Deliberately NOT scoped by risk class — this is the one integrity
18
+ * computation the whole cycle shares; risk-class-specific gate strictness
19
+ * (runner.ts) decides HOW MUCH of it each migration's completion gate
20
+ * actually checks.
21
+ */
22
+ import { createHash } from "node:crypto";
23
+ import { hashSourceFields, sourceFieldsFor } from "./source-fields.js";
24
+ export const ENVELOPE_TABLES = ["Memory", "Relationship"];
25
+ export async function computeCorpusEnvelope(getTable, now) {
26
+ const perRowHash = new Map();
27
+ const parts = [];
28
+ for (const table of ENVELOPE_TABLES) {
29
+ const fields = sourceFieldsFor(table);
30
+ const accessor = getTable(table);
31
+ for await (const row of accessor.search({})) {
32
+ const id = String(row.id ?? "");
33
+ if (!id)
34
+ continue;
35
+ const h = hashSourceFields(row, fields);
36
+ perRowHash.set(`${table}:${id}`, h);
37
+ parts.push(`${table}:${id}:${h}`);
38
+ }
39
+ }
40
+ parts.sort();
41
+ const corpusHash = createHash("sha256").update(parts.join("|")).digest("hex");
42
+ return { perRowHash, corpusHash, computedAt: now().toISOString() };
43
+ }
@@ -0,0 +1,38 @@
1
+ /**
2
+ * export.ts — content-only logical export fallback (ladder step 4):
3
+ * "Content-only logical export fallback: embeddings/indexes dominate
4
+ * data-dir size; the irreplaceable part (memory content + provenance)
5
+ * exports far smaller. If the physical snapshot doesn't fit but the content
6
+ * export does → export content, proceed (derived is recomputable by
7
+ * definition)." (flair#695, space-pressure step 4)
8
+ *
9
+ * Exports SOURCE_FIELDS ONLY (+id) — never derived fields (embedding is the
10
+ * dominant size driver this exists to skip) — as JSONL, one row per line,
11
+ * under the SAME 0700/0600 discipline as snapshot.ts.
12
+ */
13
+ import { statSync } from "node:fs";
14
+ import { join } from "node:path";
15
+ import { ensureSecureDir, writeSecureFile } from "./dir-safety.js";
16
+ import { sourceFieldsFor } from "./source-fields.js";
17
+ function sanitizeIdPart(s) {
18
+ return s.replace(/[^a-zA-Z0-9_-]/g, "_");
19
+ }
20
+ export function createContentOnlyExport(opts, deps) {
21
+ const now = deps.now();
22
+ const iso = now.toISOString().replace(/[:.]/g, "-");
23
+ const dir = join(deps.exportRoot, `${sanitizeIdPart(opts.migrationId)}-${opts.table}-${iso}`);
24
+ ensureSecureDir(dir);
25
+ const fields = sourceFieldsFor(opts.table);
26
+ const lines = opts.rows.map((row) => {
27
+ const picked = { id: row.id };
28
+ for (const f of fields)
29
+ picked[f] = row[f] ?? null;
30
+ return JSON.stringify(picked);
31
+ });
32
+ const body = lines.join("\n") + (lines.length ? "\n" : "");
33
+ const path = join(dir, `${opts.table}.jsonl`);
34
+ writeSecureFile(path, body, dir);
35
+ const metaPath = join(dir, "manifest.json");
36
+ writeSecureFile(metaPath, JSON.stringify({ migrationId: opts.migrationId, table: opts.table, fromVersion: opts.fromVersion, createdAt: now.toISOString(), rowCount: opts.rows.length }, null, 2) + "\n", dir);
37
+ return { dir, path, bytes: statSync(path).size, rowCount: opts.rows.length };
38
+ }
@@ -0,0 +1,55 @@
1
+ /**
2
+ * ledger.ts — the migration ledger OrgEvent (invariant IV: "every migration
3
+ * records an OrgEvent (initiator, versions, scope, outcome, duration) —
4
+ * migrations get decision provenance. Dogfood.")
5
+ *
6
+ * Sherlock verdict: "Ledger discloses structural metadata only — versions,
7
+ * counts, outcome, duration; never memory IDs or content summaries." The
8
+ * shape below is EXACTLY the Kern-verdict field list (migration id,
9
+ * initiator, from/to version, scope, start/end, outcome, rows
10
+ * processed/remaining, error if halted, hash envelope) — nothing else is
11
+ * ever written into `detail`.
12
+ *
13
+ * Written via an internal (no-HTTP-context) call to
14
+ * `databases.flair.OrgEvent.put()` — resolveAgentAuth(undefined) resolves
15
+ * to `{kind: "internal"}` (trusted; see resources/agent-auth.ts), the same
16
+ * pattern resources/auth-middleware.ts's backfillEmbedding() and
17
+ * resources/MemoryReindex.ts's `_reindex` path already rely on for
18
+ * server-internal writes with no HTTP caller behind them.
19
+ */
20
+ import { databases } from "@harperfast/harper";
21
+ function defaultOrgEventTable() {
22
+ return databases.flair.OrgEvent;
23
+ }
24
+ /** Structural-only detail blob — see the module doc; never memory IDs/content. */
25
+ export function buildLedgerDetail(evt) {
26
+ return JSON.stringify({
27
+ migrationId: evt.migrationId,
28
+ initiator: evt.initiator,
29
+ fromVersion: evt.fromVersion,
30
+ toVersion: evt.toVersion,
31
+ scope: evt.scope,
32
+ startedAt: evt.startedAt,
33
+ endedAt: evt.endedAt,
34
+ outcome: evt.outcome,
35
+ rowsProcessed: evt.rowsProcessed,
36
+ rowsRemaining: evt.rowsRemaining,
37
+ hashEnvelopeMatch: evt.hashEnvelopeMatch,
38
+ ...(evt.error ? { error: evt.error } : {}),
39
+ });
40
+ }
41
+ export async function writeLedgerEvent(evt, deps = {}) {
42
+ const table = deps.orgEventTable ?? defaultOrgEventTable();
43
+ const id = `migration-${evt.migrationId}-${evt.endedAt}`;
44
+ const remainingNote = evt.rowsRemaining > 0 ? `, ${evt.rowsRemaining} remaining` : "";
45
+ await table.put({
46
+ id,
47
+ authorId: "flair-migrations",
48
+ kind: "migration",
49
+ scope: evt.scope,
50
+ summary: `migration ${evt.migrationId} ${evt.outcome} (${evt.rowsProcessed} row${evt.rowsProcessed === 1 ? "" : "s"} processed${remainingNote})`,
51
+ detail: buildLedgerDetail(evt),
52
+ refId: evt.migrationId,
53
+ createdAt: evt.endedAt,
54
+ });
55
+ }