@tpsdev-ai/flair 0.20.1 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -11
- package/SECURITY.md +52 -20
- package/config.yaml +32 -5
- package/dist/cli.js +2235 -260
- package/dist/deploy.js +96 -15
- package/dist/doctor-client.js +312 -0
- package/dist/fleet-presence.js +98 -0
- package/dist/fleet-verify.js +291 -0
- package/dist/install/clients.js +18 -0
- package/dist/mcp-client-assertion.js +364 -0
- package/dist/probe.js +97 -0
- package/dist/rem/restore.js +1 -1
- package/dist/rem/runner.js +82 -12
- package/dist/resources/Admin.js +1 -1
- package/dist/resources/AdminConnectors.js +1 -1
- package/dist/resources/AdminDashboard.js +1 -1
- package/dist/resources/AdminIdp.js +1 -1
- package/dist/resources/AdminInstance.js +1 -1
- package/dist/resources/AdminMemory.js +2 -2
- package/dist/resources/AdminPrincipals.js +1 -1
- package/dist/resources/Agent.js +14 -0
- package/dist/resources/AgentCard.js +1 -1
- package/dist/resources/AgentSeed.js +1 -1
- package/dist/resources/AttentionQuery.js +356 -0
- package/dist/resources/Credential.js +11 -1
- package/dist/resources/Federation.js +121 -2
- package/dist/resources/Integration.js +1 -1
- package/dist/resources/MCPClientMetadata.js +88 -0
- package/dist/resources/Memory.js +127 -22
- package/dist/resources/MemoryBootstrap.js +441 -85
- package/dist/resources/MemoryGrant.js +1 -1
- package/dist/resources/MemoryReflect.js +105 -49
- package/dist/resources/MemoryUsage.js +104 -0
- package/dist/resources/OAuth.js +69 -5
- package/dist/resources/OrgEvent.js +11 -0
- package/dist/resources/OrgEventCatchup.js +1 -1
- package/dist/resources/Presence.js +268 -17
- package/dist/resources/RecordUsage.js +230 -0
- package/dist/resources/Relationship.js +108 -26
- package/dist/resources/SemanticSearch.js +88 -320
- package/dist/resources/SkillScan.js +15 -0
- package/dist/resources/Soul.js +14 -0
- package/dist/resources/WorkspaceLatest.js +1 -1
- package/dist/resources/WorkspaceState.js +12 -1
- package/dist/resources/agent-auth.js +61 -3
- package/dist/resources/agentcard-fields.js +2 -2
- package/dist/resources/auth-middleware.js +42 -14
- package/dist/resources/bm25-filter.js +1 -1
- package/dist/resources/bm25.js +13 -7
- package/dist/resources/collision-lib.js +157 -0
- package/dist/resources/dedup.js +2 -2
- package/dist/resources/ed25519-auth.js +2 -2
- package/dist/resources/embeddings-boot.js +149 -0
- package/dist/resources/embeddings-provider.js +364 -106
- package/dist/resources/entity-vocab.js +139 -0
- package/dist/resources/federation-nonce-store.js +195 -0
- package/dist/resources/health.js +97 -6
- package/dist/resources/instance-identity.js +53 -0
- package/dist/resources/mcp-client-metadata-fields.js +112 -0
- package/dist/resources/mcp-handler.js +1 -1
- package/dist/resources/mcp-tools.js +88 -10
- package/dist/resources/memory-bootstrap-lib.js +1 -1
- package/dist/resources/memory-read-scope.js +58 -71
- package/dist/resources/memory-reflect-lib.js +289 -0
- package/dist/resources/memory-visibility.js +37 -0
- package/dist/resources/migration-boot.js +143 -0
- package/dist/resources/migrations/dir-safety.js +71 -0
- package/dist/resources/migrations/embedding-stamp.js +178 -0
- package/dist/resources/migrations/envelope.js +43 -0
- package/dist/resources/migrations/export.js +38 -0
- package/dist/resources/migrations/ledger.js +55 -0
- package/dist/resources/migrations/lock.js +154 -0
- package/dist/resources/migrations/progress.js +31 -0
- package/dist/resources/migrations/registry.js +36 -0
- package/dist/resources/migrations/risk-policy.js +23 -0
- package/dist/resources/migrations/runner.js +456 -0
- package/dist/resources/migrations/snapshot.js +127 -0
- package/dist/resources/migrations/source-fields.js +93 -0
- package/dist/resources/migrations/space.js +73 -0
- package/dist/resources/migrations/state.js +64 -0
- package/dist/resources/migrations/status.js +39 -0
- package/dist/resources/migrations/synthetic-test-migration.js +93 -0
- package/dist/resources/migrations/types.js +9 -0
- package/dist/resources/presence-internal.js +29 -0
- package/dist/resources/provenance.js +50 -0
- package/dist/resources/rate-limiter.js +8 -1
- package/dist/resources/scoring.js +124 -5
- package/dist/resources/semantic-retrieval-core.js +317 -0
- package/dist/version-check.js +167 -0
- package/dist/version-handshake.js +122 -0
- package/package.json +4 -5
- package/schemas/agent.graphql +7 -0
- package/schemas/event.graphql +5 -0
- package/schemas/federation.graphql +12 -0
- package/schemas/memory.graphql +82 -0
- package/schemas/schema.graphql +5 -43
- package/schemas/workspace.graphql +3 -0
- package/dist/resources/IngestEvents.js +0 -162
- package/dist/resources/IssueTokens.js +0 -19
- package/dist/resources/ObsAgentSnapshot.js +0 -13
- package/dist/resources/ObsEventFeed.js +0 -13
- package/dist/resources/ObsOffice.js +0 -19
- package/dist/resources/ObservationCenter.js +0 -23
- package/ui/observation-center.html +0 -385
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* migration-boot.ts — the boot-keyed trigger (flair#695
|
|
3
|
+
* §A, Kern verdict): "Envelope ASYNC after ready — boot serves immediately
|
|
4
|
+
* on the old shape; pre-hash runs async; migration deferred until it
|
|
5
|
+
* completes; health shows 'pre-flight integrity check in progress'."
|
|
6
|
+
*
|
|
7
|
+
* This is a plain (non-Resource) module — same shape as
|
|
8
|
+
* resources/embeddings-provider.ts / resources/table-helpers.ts / etc. — so
|
|
9
|
+
* Harper's `jsResource: files: dist/resources/*.js` loader (config.yaml)
|
|
10
|
+
* imports it at boot like every other flat file under resources/, running
|
|
11
|
+
* its top-level side effect exactly once per process. It exports no
|
|
12
|
+
* Resource subclass — there is no HTTP endpoint here, only the trigger.
|
|
13
|
+
*
|
|
14
|
+
* Timing: `scheduleMigrationBoot()` defers the actual cycle via
|
|
15
|
+
* `setImmediate`, which runs after the current synchronous phase (module
|
|
16
|
+
* loading / resource registration) yields to the event loop — in practice,
|
|
17
|
+
* after Harper's HTTP listener is already accepting connections, so the
|
|
18
|
+
* server is serving on the OLD shape before any migration write happens
|
|
19
|
+
* (the #687 boot-win property this preserves). As an additional guard
|
|
20
|
+
* against Harper-internal load-ordering this file has no visibility into,
|
|
21
|
+
* the deferred callback ALSO polls for `databases.flair.Memory`/`Relationship`
|
|
22
|
+
* actually being live table accessors before invoking the runner — cheap,
|
|
23
|
+
* bounded, and self-healing if the very first check is too early.
|
|
24
|
+
*
|
|
25
|
+
* `runMigrationCycle` itself never throws (see runner.ts's module doc) —
|
|
26
|
+
* the `.catch()` below is pure defense-in-depth so a bug there can never
|
|
27
|
+
* take down the process either.
|
|
28
|
+
*/
|
|
29
|
+
import { databases } from "@harperfast/harper";
|
|
30
|
+
import { homedir } from "node:os";
|
|
31
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
32
|
+
import { join, dirname } from "node:path";
|
|
33
|
+
import { fileURLToPath } from "node:url";
|
|
34
|
+
import { buildRegistry } from "./migrations/registry.js";
|
|
35
|
+
import { runMigrationCycle } from "./migrations/runner.js";
|
|
36
|
+
import { seedIdleProgress } from "./migrations/progress.js";
|
|
37
|
+
import { getMode } from "./embeddings-provider.js";
|
|
38
|
+
/** Same dataDir resolution as resources/health.ts's disk section. */
|
|
39
|
+
export function resolveMigrationDataDir() {
|
|
40
|
+
return process.env.HDB_ROOT ?? join(homedir(), ".flair", "data");
|
|
41
|
+
}
|
|
42
|
+
/** Same "resolve the running package's own version" idiom as resources/health.ts. */
|
|
43
|
+
export function resolveRunningVersion() {
|
|
44
|
+
try {
|
|
45
|
+
const here = dirname(fileURLToPath(import.meta.url));
|
|
46
|
+
const candidates = [join(here, "..", "..", "package.json"), join(here, "..", "package.json")];
|
|
47
|
+
for (const p of candidates) {
|
|
48
|
+
if (existsSync(p)) {
|
|
49
|
+
const pkg = JSON.parse(readFileSync(p, "utf-8"));
|
|
50
|
+
if (pkg.version)
|
|
51
|
+
return pkg.version;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
/* fall through */
|
|
57
|
+
}
|
|
58
|
+
return process.env.npm_package_version ?? "dev";
|
|
59
|
+
}
|
|
60
|
+
function getTable(table) {
|
|
61
|
+
return databases.flair[table];
|
|
62
|
+
}
|
|
63
|
+
async function waitForTablesReady(maxWaitMs = 30_000, intervalMs = 200) {
|
|
64
|
+
const deadline = Date.now() + maxWaitMs;
|
|
65
|
+
for (;;) {
|
|
66
|
+
try {
|
|
67
|
+
const flair = databases?.flair;
|
|
68
|
+
const mem = flair?.Memory;
|
|
69
|
+
const rel = flair?.Relationship;
|
|
70
|
+
if (typeof mem?.search === "function" && typeof rel?.search === "function")
|
|
71
|
+
return true;
|
|
72
|
+
}
|
|
73
|
+
catch {
|
|
74
|
+
/* keep polling */
|
|
75
|
+
}
|
|
76
|
+
if (Date.now() >= deadline)
|
|
77
|
+
return false;
|
|
78
|
+
await new Promise((r) => setTimeout(r, intervalMs));
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Gives the embeddings engine a bounded window to finish its own boot probe
|
|
83
|
+
* (resources/embeddings-provider.ts: PROBE_TIMEOUT_MS = 8s) before running
|
|
84
|
+
* migrations. Root-cause fix for a real race found while building the
|
|
85
|
+
* embedding-stamp integration test: this trigger fires very early (right
|
|
86
|
+
* after the Memory/Relationship tables exist), which can beat the
|
|
87
|
+
* embeddings engine's own async model-load — a Memory.put() regen attempted
|
|
88
|
+
* during that window silently fails (getEmbedding() catches and returns
|
|
89
|
+
* null), leaving a row's embeddingModel null instead of freshly stamped.
|
|
90
|
+
* `getMode()==="local"` breaks out early on the common case (embeddings
|
|
91
|
+
* already warm); otherwise this waits up to ~8.5s (a hair over the probe's
|
|
92
|
+
* own timeout, so whatever getMode() reports by then is genuinely settled,
|
|
93
|
+
* not just "haven't checked yet") and proceeds regardless — a migration
|
|
94
|
+
* must never block boot indefinitely on this, and embedding-stamp's own
|
|
95
|
+
* pending-condition (OR of not_equal + equals-null — see that file) still
|
|
96
|
+
* self-heals a stray null on the NEXT boot even if this window isn't enough.
|
|
97
|
+
*/
|
|
98
|
+
async function waitForEmbeddingsSettled(maxWaitMs = 8_500, intervalMs = 150) {
|
|
99
|
+
const deadline = Date.now() + maxWaitMs;
|
|
100
|
+
while (Date.now() < deadline) {
|
|
101
|
+
if (getMode() === "local")
|
|
102
|
+
return;
|
|
103
|
+
await new Promise((r) => setTimeout(r, intervalMs));
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
let scheduled = false;
|
|
107
|
+
export function scheduleMigrationBoot() {
|
|
108
|
+
if (scheduled)
|
|
109
|
+
return;
|
|
110
|
+
scheduled = true;
|
|
111
|
+
const registry = buildRegistry();
|
|
112
|
+
seedIdleProgress(registry.list().map((m) => m.id));
|
|
113
|
+
setImmediate(() => {
|
|
114
|
+
void (async () => {
|
|
115
|
+
const ready = await waitForTablesReady();
|
|
116
|
+
if (!ready) {
|
|
117
|
+
console.error("[flair-migrations] Memory/Relationship tables never became ready — skipping this boot's migration cycle (will retry next boot)");
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
await waitForEmbeddingsSettled();
|
|
121
|
+
try {
|
|
122
|
+
await runMigrationCycle({
|
|
123
|
+
registry,
|
|
124
|
+
getTable,
|
|
125
|
+
dataDir: resolveMigrationDataDir(),
|
|
126
|
+
runningVersion: resolveRunningVersion(),
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
catch (err) {
|
|
130
|
+
// Defense-in-depth only — runMigrationCycle is documented to never
|
|
131
|
+
// throw. A boot-path exception must never surface here regardless.
|
|
132
|
+
console.error(`[flair-migrations] unexpected error from runMigrationCycle: ${err?.message ?? String(err)}`);
|
|
133
|
+
}
|
|
134
|
+
})();
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
// Test-only reset so a unit/integration test can re-trigger scheduling
|
|
138
|
+
// within the same process (never used in production — a real process only
|
|
139
|
+
// ever boots once).
|
|
140
|
+
export function _resetMigrationBootForTests() {
|
|
141
|
+
scheduled = false;
|
|
142
|
+
}
|
|
143
|
+
scheduleMigrationBoot();
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dir-safety.ts — 0700 directory enforcement for migration snapshot/export
|
|
3
|
+
* dirs (Sherlock verdict: "0700 via stat-verify after creation (umask can
|
|
4
|
+
* override mkdir mode) and before writes; refuse with the path + actual
|
|
5
|
+
* perms named in the error." — "the safety mechanism must not become the
|
|
6
|
+
* exfil surface... export dir = highest-value target on the system.")
|
|
7
|
+
*
|
|
8
|
+
* `ensureSecureDir` creates-or-verifies a directory is 0700: it mkdirs with
|
|
9
|
+
* mode 0700, then explicitly `chmodSync`s to 0700 (covers a permissive
|
|
10
|
+
* umask silently widening mkdir's mode argument), then stat-verifies the
|
|
11
|
+
* result. If the directory STILL isn't 0700 after the explicit chmod
|
|
12
|
+
* (unusual filesystem/ACL), it refuses loudly rather than silently
|
|
13
|
+
* proceeding into a world-readable directory — naming the exact path and
|
|
14
|
+
* actual octal perms in the thrown error, per Sherlock's wording.
|
|
15
|
+
*
|
|
16
|
+
* `verifySecureDir` is exported separately so callers can (and must)
|
|
17
|
+
* re-verify immediately before EACH write inside the directory, not only at
|
|
18
|
+
* creation time (Kern: "Perms re-verified at write time, not only
|
|
19
|
+
* creation.") — a directory can be re-permissioned by something else on the
|
|
20
|
+
* host between creation and a later write in a long-running batch loop.
|
|
21
|
+
*/
|
|
22
|
+
import { existsSync, mkdirSync, chmodSync, statSync, writeFileSync } from "node:fs";
|
|
23
|
+
export class UnsafeDirectoryError extends Error {
|
|
24
|
+
path;
|
|
25
|
+
actualMode;
|
|
26
|
+
constructor(path, actualMode) {
|
|
27
|
+
super(`refusing to use ${path}: expected 0700, found ${actualMode.toString(8).padStart(4, "0")} ` +
|
|
28
|
+
`(group/other-accessible) — fix permissions (chmod 700 ${path}) or point the migration ` +
|
|
29
|
+
`at a different directory`);
|
|
30
|
+
this.path = path;
|
|
31
|
+
this.actualMode = actualMode;
|
|
32
|
+
this.name = "UnsafeDirectoryError";
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
/** Throws UnsafeDirectoryError if `dir`'s mode has any group/other bits set. */
|
|
36
|
+
export function verifySecureDir(dir) {
|
|
37
|
+
const st = statSync(dir);
|
|
38
|
+
const mode = st.mode & 0o777;
|
|
39
|
+
if ((mode & 0o077) !== 0) {
|
|
40
|
+
throw new UnsafeDirectoryError(dir, mode);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Ensures `dir` exists and is 0700. Idempotent — safe to call before every
|
|
45
|
+
* write. Never silently tolerates a world/group-readable result: attempts
|
|
46
|
+
* one explicit chmod(0700) remediation (covers the umask-widened-mkdir
|
|
47
|
+
* case), then re-verifies and throws if it's still not compliant.
|
|
48
|
+
*/
|
|
49
|
+
export function ensureSecureDir(dir) {
|
|
50
|
+
if (!existsSync(dir)) {
|
|
51
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
52
|
+
}
|
|
53
|
+
try {
|
|
54
|
+
verifySecureDir(dir);
|
|
55
|
+
}
|
|
56
|
+
catch (err) {
|
|
57
|
+
if (!(err instanceof UnsafeDirectoryError))
|
|
58
|
+
throw err;
|
|
59
|
+
chmodSync(dir, 0o700);
|
|
60
|
+
verifySecureDir(dir); // throws again (loudly) if the chmod didn't stick
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Writes `contents` to `path` at 0600, re-verifying the CONTAINING
|
|
65
|
+
* directory is still 0700 immediately before the write (per Kern's
|
|
66
|
+
* write-time re-verification requirement above).
|
|
67
|
+
*/
|
|
68
|
+
export function writeSecureFile(path, contents, dir) {
|
|
69
|
+
verifySecureDir(dir);
|
|
70
|
+
writeFileSync(path, contents, { mode: 0o600 });
|
|
71
|
+
}
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* embedding-stamp.ts — the first registered migration (per the spec):
|
|
3
|
+
* re-embeds every Memory row whose `embeddingModel` doesn't match
|
|
4
|
+
* `getModelId()`. riskClass 'derived-only' — embeddings are recomputable
|
|
5
|
+
* from `content` (SOURCE_FIELDS), never the only copy of anything
|
|
6
|
+
* (invariant I), so this is the cheapest posture: metadata-only snapshot,
|
|
7
|
+
* no content-hash gate (row-count + stamp convergence only).
|
|
8
|
+
*
|
|
9
|
+
* THE GATE (`EMBEDDING_PREFIXES_ENABLED` in resources/embeddings-provider.ts)
|
|
10
|
+
* is now ON (flair#504, flipped and re-baselined through the ratchet gate),
|
|
11
|
+
* so `getModelId()` returns `<base>+searchprefix` — every row written before
|
|
12
|
+
* this flip was stamped with the bare base id, so it now reads as stale and
|
|
13
|
+
* gets picked up by this migration on the next boot that reaches it. This is
|
|
14
|
+
* this migration's first real payload: before the flip, `getModelId()`
|
|
15
|
+
* returned the bare base id unconditionally, so this migration — live and
|
|
16
|
+
* always-registered from the day it shipped — had nothing to detect. That
|
|
17
|
+
* was intentional groundwork, not dead code: proving the detect/re-embed
|
|
18
|
+
* mechanism end-to-end (this file, `test/integration/
|
|
19
|
+
* migrations-embedding-stamp-e2e.test.ts`) before it ever had real work to
|
|
20
|
+
* do was the point.
|
|
21
|
+
*
|
|
22
|
+
* Reuses Memory's OWN regen branch — never duplicates embedding logic —
|
|
23
|
+
* via the SAME mechanism `flair reembed` (src/cli.ts) already uses in
|
|
24
|
+
* production: a genuine `PUT /Memory/:id` HTTP request (admin-authenticated
|
|
25
|
+
* loopback), not an in-process call on `databases.flair.Memory` directly.
|
|
26
|
+
*
|
|
27
|
+
* THIS IS LOAD-BEARING, confirmed empirically while building
|
|
28
|
+
* test/integration/migrations-embedding-stamp-e2e.test.ts against real
|
|
29
|
+
* Harper: `databases.flair.Memory` (the property Harper exposes to OTHER
|
|
30
|
+
* modules, imported the same way resources/health.ts / MemoryReindex.ts /
|
|
31
|
+
* etc. already do for READS) resolves to the RAW underlying table, not the
|
|
32
|
+
* resources/Memory.ts SUBCLASS that carries the actual regen/dedup/auth
|
|
33
|
+
* logic — that subclass is only reachable via Harper's own REST dispatch
|
|
34
|
+
* for a genuine `/Memory` request. Calling `.put()` on the raw reference
|
|
35
|
+
* writes the record fine (confirmed: fields land exactly as passed) but
|
|
36
|
+
* silently skips every override in resources/Memory.ts, INCLUDING the
|
|
37
|
+
* `if (content.content && !content.embedding) { regenerate } ` branch this
|
|
38
|
+
* migration exists to trigger. Reads (`.search()`/`.get()`, used below for
|
|
39
|
+
* detect/countPending/candidate-selection) are unaffected by this — only
|
|
40
|
+
* `.put()` needs the real dispatch, hence the loopback HTTP call for
|
|
41
|
+
* exactly that one step.
|
|
42
|
+
*
|
|
43
|
+
* Query correctness (also verified against real Harper): Harper's
|
|
44
|
+
* `not_equal` comparator only matches rows where the attribute holds an
|
|
45
|
+
* EXPLICIT value (including explicit `null`) — a row where the attribute
|
|
46
|
+
* was NEVER SET AT ALL (`getIndexedValues()` returns `undefined` for a
|
|
47
|
+
* truly-absent property, never indexed regardless of `indexNulls`) is
|
|
48
|
+
* invisible to ANY condition-based query, not just `not_equal`. Clearing to
|
|
49
|
+
* `null` (never `undefined`) on write is therefore load-bearing: if the
|
|
50
|
+
* regen HTTP call fails (engine not yet warmed up, transient failure,
|
|
51
|
+
* admin credential unavailable this cycle), the row must land back in an
|
|
52
|
+
* EXPLICIT-null state — queryable and retried on the next boot — never a
|
|
53
|
+
* truly-absent one that would be permanently invisible to this migration
|
|
54
|
+
* again. The pending condition is an OR of `not_equal <current>` (catches a
|
|
55
|
+
* stale non-null model string) and `equals null` (catches that
|
|
56
|
+
* explicit-null state) — together they catch every state this migration's
|
|
57
|
+
* OWN writes can ever produce. A row whose `embeddingModel` was NEVER
|
|
58
|
+
* touched by anything (truly absent from its very first write — only
|
|
59
|
+
* possible if the embeddings engine was down for that entire write) is a
|
|
60
|
+
* known, narrow gap this bounded query cannot see;
|
|
61
|
+
* resources/migration-boot.ts mitigates the common case by waiting for the
|
|
62
|
+
* embeddings engine to settle before running migrations at all.
|
|
63
|
+
*/
|
|
64
|
+
import { databases } from "@harperfast/harper";
|
|
65
|
+
import { getModelId } from "../embeddings-provider.js";
|
|
66
|
+
function defaultMemoryTable() {
|
|
67
|
+
return databases.flair.Memory;
|
|
68
|
+
}
|
|
69
|
+
export const EMBEDDING_STAMP_ID = "embedding-stamp";
|
|
70
|
+
const REGEN_HTTP_TIMEOUT_MS = 20_000; // a real embedding compute can be slow on constrained hardware
|
|
71
|
+
/** Same admin-password resolution as resources/auth-middleware.ts's getAdminPass(). */
|
|
72
|
+
function resolveAdminAuthHeader() {
|
|
73
|
+
const pass = process.env.HDB_ADMIN_PASSWORD ?? process.env.FLAIR_ADMIN_PASSWORD;
|
|
74
|
+
if (!pass)
|
|
75
|
+
return null;
|
|
76
|
+
return "Basic " + Buffer.from(`admin:${pass}`).toString("base64");
|
|
77
|
+
}
|
|
78
|
+
/** Same HTTP_PORT env resolution src/cli.ts sets on every Harper spawn (see that file's grep for HTTP_PORT). */
|
|
79
|
+
function resolveSelfBaseUrl() {
|
|
80
|
+
const port = process.env.HTTP_PORT ?? "9926";
|
|
81
|
+
return `http://127.0.0.1:${port}`;
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Triggers Memory.put()'s regen branch via a genuine loopback HTTP PUT —
|
|
85
|
+
* the ONLY reliable way to reach resources/Memory.ts's subclass logic (see
|
|
86
|
+
* module doc). Returns true iff the request succeeded (2xx); NEVER throws
|
|
87
|
+
* — a failure just leaves the row in its current (queryable, explicit-null
|
|
88
|
+
* or stale-string) state for the next attempt.
|
|
89
|
+
*/
|
|
90
|
+
async function regenViaHttpPut(id, existing, fetchImpl) {
|
|
91
|
+
const authHeader = resolveAdminAuthHeader();
|
|
92
|
+
if (!authHeader)
|
|
93
|
+
return false; // no admin credential available this cycle — retried next boot
|
|
94
|
+
try {
|
|
95
|
+
const res = await fetchImpl(`${resolveSelfBaseUrl()}/Memory/${encodeURIComponent(id)}`, {
|
|
96
|
+
method: "PUT",
|
|
97
|
+
headers: { "Content-Type": "application/json", Authorization: authHeader },
|
|
98
|
+
body: JSON.stringify({ ...existing, embedding: null, embeddingModel: null }),
|
|
99
|
+
signal: AbortSignal.timeout(REGEN_HTTP_TIMEOUT_MS),
|
|
100
|
+
});
|
|
101
|
+
return res.ok;
|
|
102
|
+
}
|
|
103
|
+
catch {
|
|
104
|
+
return false;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* `getTable`/`getCurrentModelId`/`regen` are injectable so unit tests can
|
|
109
|
+
* exercise this migration's full detect/countPending/run logic against an
|
|
110
|
+
* in-memory fake table and a fake regen function (no real Harper, no real
|
|
111
|
+
* HTTP needed) — matching the mocking technique used throughout
|
|
112
|
+
* test/unit/*.ts (e.g. test/unit/instance-identity.test.ts). The real
|
|
113
|
+
* loopback-HTTP regen path is exercised for real in
|
|
114
|
+
* test/integration/migrations-embedding-stamp-e2e.test.ts.
|
|
115
|
+
*/
|
|
116
|
+
export function createEmbeddingStampMigration(getTable = defaultMemoryTable, getCurrentModelId = getModelId, regen = (id, existing) => regenViaHttpPut(id, existing, fetch)) {
|
|
117
|
+
function staleCondition() {
|
|
118
|
+
// OR-combined: `not_equal <current>` catches a stale non-null model
|
|
119
|
+
// string; `equals null` catches the explicit-null state this
|
|
120
|
+
// migration's own writes leave behind on a failed regen (see the
|
|
121
|
+
// module doc above — Harper's index never sees a TRULY ABSENT
|
|
122
|
+
// property, only an explicit null).
|
|
123
|
+
return [
|
|
124
|
+
{
|
|
125
|
+
operator: "or",
|
|
126
|
+
conditions: [
|
|
127
|
+
{ attribute: "embeddingModel", comparator: "not_equal", value: getCurrentModelId() },
|
|
128
|
+
{ attribute: "embeddingModel", comparator: "equals", value: null },
|
|
129
|
+
],
|
|
130
|
+
},
|
|
131
|
+
];
|
|
132
|
+
}
|
|
133
|
+
return {
|
|
134
|
+
id: EMBEDDING_STAMP_ID,
|
|
135
|
+
riskClass: "derived-only",
|
|
136
|
+
affectsTables: ["Memory"],
|
|
137
|
+
async detect() {
|
|
138
|
+
const table = getTable();
|
|
139
|
+
for await (const _row of table.search({ conditions: staleCondition(), limit: 1 })) {
|
|
140
|
+
return true;
|
|
141
|
+
}
|
|
142
|
+
return false;
|
|
143
|
+
},
|
|
144
|
+
async countPending() {
|
|
145
|
+
const table = getTable();
|
|
146
|
+
let n = 0;
|
|
147
|
+
for await (const _row of table.search({ conditions: staleCondition() }))
|
|
148
|
+
n++;
|
|
149
|
+
return n;
|
|
150
|
+
},
|
|
151
|
+
async run(batchSize) {
|
|
152
|
+
const table = getTable();
|
|
153
|
+
const current = getCurrentModelId();
|
|
154
|
+
const candidates = [];
|
|
155
|
+
for await (const row of table.search({ conditions: staleCondition(), limit: batchSize })) {
|
|
156
|
+
candidates.push(row);
|
|
157
|
+
}
|
|
158
|
+
const touchedIds = [];
|
|
159
|
+
for (const row of candidates) {
|
|
160
|
+
const id = String(row.id ?? "");
|
|
161
|
+
if (!id)
|
|
162
|
+
continue;
|
|
163
|
+
const existing = await table.get(id);
|
|
164
|
+
if (!existing)
|
|
165
|
+
continue; // deleted since the search above — nothing to fix
|
|
166
|
+
if (existing.embeddingModel === current)
|
|
167
|
+
continue; // already stamped by a concurrent runner — idempotent skip
|
|
168
|
+
const ok = await regen(id, existing);
|
|
169
|
+
if (ok)
|
|
170
|
+
touchedIds.push(id);
|
|
171
|
+
// A failed regen leaves the row untouched (still matching
|
|
172
|
+
// staleCondition — retried next batch/boot), never partially
|
|
173
|
+
// written or marked done.
|
|
174
|
+
}
|
|
175
|
+
return { processed: touchedIds.length, touchedIds };
|
|
176
|
+
},
|
|
177
|
+
};
|
|
178
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* envelope.ts — the corpus-wide integrity envelope: "full-corpus
|
|
3
|
+
* hash-of-source-field-hashes computed before first write and after
|
|
4
|
+
* completion; must match" (flair#695 invariant IV /
|
|
5
|
+
* Kern verdict), computed ONCE per boot cycle (K&S: "Envelope ASYNC after
|
|
6
|
+
* ready — boot serves immediately on the old shape; pre-hash runs async;
|
|
7
|
+
* migration deferred until it completes").
|
|
8
|
+
*
|
|
9
|
+
* `computeCorpusEnvelope` walks BOTH SOURCE_FIELDS tables (Memory,
|
|
10
|
+
* Relationship) once and returns both the single aggregate `corpusHash`
|
|
11
|
+
* (what schema-additive's full-envelope gate compares pre/post) AND the
|
|
12
|
+
* full `perRowHash` breakdown (`${table}:${id}` -> hash) — the latter isn't
|
|
13
|
+
* used by this module directly, but keeping the breakdown around lets a
|
|
14
|
+
* caller answer "did THIS specific row's source fields change" without a
|
|
15
|
+
* second full-corpus pass, if ever needed for diagnostics.
|
|
16
|
+
*
|
|
17
|
+
* Deliberately NOT scoped by risk class — this is the one integrity
|
|
18
|
+
* computation the whole cycle shares; risk-class-specific gate strictness
|
|
19
|
+
* (runner.ts) decides HOW MUCH of it each migration's completion gate
|
|
20
|
+
* actually checks.
|
|
21
|
+
*/
|
|
22
|
+
import { createHash } from "node:crypto";
|
|
23
|
+
import { hashSourceFields, sourceFieldsFor } from "./source-fields.js";
|
|
24
|
+
export const ENVELOPE_TABLES = ["Memory", "Relationship"];
|
|
25
|
+
export async function computeCorpusEnvelope(getTable, now) {
|
|
26
|
+
const perRowHash = new Map();
|
|
27
|
+
const parts = [];
|
|
28
|
+
for (const table of ENVELOPE_TABLES) {
|
|
29
|
+
const fields = sourceFieldsFor(table);
|
|
30
|
+
const accessor = getTable(table);
|
|
31
|
+
for await (const row of accessor.search({})) {
|
|
32
|
+
const id = String(row.id ?? "");
|
|
33
|
+
if (!id)
|
|
34
|
+
continue;
|
|
35
|
+
const h = hashSourceFields(row, fields);
|
|
36
|
+
perRowHash.set(`${table}:${id}`, h);
|
|
37
|
+
parts.push(`${table}:${id}:${h}`);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
parts.sort();
|
|
41
|
+
const corpusHash = createHash("sha256").update(parts.join("|")).digest("hex");
|
|
42
|
+
return { perRowHash, corpusHash, computedAt: now().toISOString() };
|
|
43
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* export.ts — content-only logical export fallback (ladder step 4):
|
|
3
|
+
* "Content-only logical export fallback: embeddings/indexes dominate
|
|
4
|
+
* data-dir size; the irreplaceable part (memory content + provenance)
|
|
5
|
+
* exports far smaller. If the physical snapshot doesn't fit but the content
|
|
6
|
+
* export does → export content, proceed (derived is recomputable by
|
|
7
|
+
* definition)." (flair#695, space-pressure step 4)
|
|
8
|
+
*
|
|
9
|
+
* Exports SOURCE_FIELDS ONLY (+id) — never derived fields (embedding is the
|
|
10
|
+
* dominant size driver this exists to skip) — as JSONL, one row per line,
|
|
11
|
+
* under the SAME 0700/0600 discipline as snapshot.ts.
|
|
12
|
+
*/
|
|
13
|
+
import { statSync } from "node:fs";
|
|
14
|
+
import { join } from "node:path";
|
|
15
|
+
import { ensureSecureDir, writeSecureFile } from "./dir-safety.js";
|
|
16
|
+
import { sourceFieldsFor } from "./source-fields.js";
|
|
17
|
+
function sanitizeIdPart(s) {
|
|
18
|
+
return s.replace(/[^a-zA-Z0-9_-]/g, "_");
|
|
19
|
+
}
|
|
20
|
+
export function createContentOnlyExport(opts, deps) {
|
|
21
|
+
const now = deps.now();
|
|
22
|
+
const iso = now.toISOString().replace(/[:.]/g, "-");
|
|
23
|
+
const dir = join(deps.exportRoot, `${sanitizeIdPart(opts.migrationId)}-${opts.table}-${iso}`);
|
|
24
|
+
ensureSecureDir(dir);
|
|
25
|
+
const fields = sourceFieldsFor(opts.table);
|
|
26
|
+
const lines = opts.rows.map((row) => {
|
|
27
|
+
const picked = { id: row.id };
|
|
28
|
+
for (const f of fields)
|
|
29
|
+
picked[f] = row[f] ?? null;
|
|
30
|
+
return JSON.stringify(picked);
|
|
31
|
+
});
|
|
32
|
+
const body = lines.join("\n") + (lines.length ? "\n" : "");
|
|
33
|
+
const path = join(dir, `${opts.table}.jsonl`);
|
|
34
|
+
writeSecureFile(path, body, dir);
|
|
35
|
+
const metaPath = join(dir, "manifest.json");
|
|
36
|
+
writeSecureFile(metaPath, JSON.stringify({ migrationId: opts.migrationId, table: opts.table, fromVersion: opts.fromVersion, createdAt: now.toISOString(), rowCount: opts.rows.length }, null, 2) + "\n", dir);
|
|
37
|
+
return { dir, path, bytes: statSync(path).size, rowCount: opts.rows.length };
|
|
38
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ledger.ts — the migration ledger OrgEvent (invariant IV: "every migration
|
|
3
|
+
* records an OrgEvent (initiator, versions, scope, outcome, duration) —
|
|
4
|
+
* migrations get decision provenance. Dogfood.")
|
|
5
|
+
*
|
|
6
|
+
* Sherlock verdict: "Ledger discloses structural metadata only — versions,
|
|
7
|
+
* counts, outcome, duration; never memory IDs or content summaries." The
|
|
8
|
+
* shape below is EXACTLY the Kern-verdict field list (migration id,
|
|
9
|
+
* initiator, from/to version, scope, start/end, outcome, rows
|
|
10
|
+
* processed/remaining, error if halted, hash envelope) — nothing else is
|
|
11
|
+
* ever written into `detail`.
|
|
12
|
+
*
|
|
13
|
+
* Written via an internal (no-HTTP-context) call to
|
|
14
|
+
* `databases.flair.OrgEvent.put()` — resolveAgentAuth(undefined) resolves
|
|
15
|
+
* to `{kind: "internal"}` (trusted; see resources/agent-auth.ts), the same
|
|
16
|
+
* pattern resources/auth-middleware.ts's backfillEmbedding() and
|
|
17
|
+
* resources/MemoryReindex.ts's `_reindex` path already rely on for
|
|
18
|
+
* server-internal writes with no HTTP caller behind them.
|
|
19
|
+
*/
|
|
20
|
+
import { databases } from "@harperfast/harper";
|
|
21
|
+
function defaultOrgEventTable() {
|
|
22
|
+
return databases.flair.OrgEvent;
|
|
23
|
+
}
|
|
24
|
+
/** Structural-only detail blob — see the module doc; never memory IDs/content. */
|
|
25
|
+
export function buildLedgerDetail(evt) {
|
|
26
|
+
return JSON.stringify({
|
|
27
|
+
migrationId: evt.migrationId,
|
|
28
|
+
initiator: evt.initiator,
|
|
29
|
+
fromVersion: evt.fromVersion,
|
|
30
|
+
toVersion: evt.toVersion,
|
|
31
|
+
scope: evt.scope,
|
|
32
|
+
startedAt: evt.startedAt,
|
|
33
|
+
endedAt: evt.endedAt,
|
|
34
|
+
outcome: evt.outcome,
|
|
35
|
+
rowsProcessed: evt.rowsProcessed,
|
|
36
|
+
rowsRemaining: evt.rowsRemaining,
|
|
37
|
+
hashEnvelopeMatch: evt.hashEnvelopeMatch,
|
|
38
|
+
...(evt.error ? { error: evt.error } : {}),
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
export async function writeLedgerEvent(evt, deps = {}) {
|
|
42
|
+
const table = deps.orgEventTable ?? defaultOrgEventTable();
|
|
43
|
+
const id = `migration-${evt.migrationId}-${evt.endedAt}`;
|
|
44
|
+
const remainingNote = evt.rowsRemaining > 0 ? `, ${evt.rowsRemaining} remaining` : "";
|
|
45
|
+
await table.put({
|
|
46
|
+
id,
|
|
47
|
+
authorId: "flair-migrations",
|
|
48
|
+
kind: "migration",
|
|
49
|
+
scope: evt.scope,
|
|
50
|
+
summary: `migration ${evt.migrationId} ${evt.outcome} (${evt.rowsProcessed} row${evt.rowsProcessed === 1 ? "" : "s"} processed${remainingNote})`,
|
|
51
|
+
detail: buildLedgerDetail(evt),
|
|
52
|
+
refId: evt.migrationId,
|
|
53
|
+
createdAt: evt.endedAt,
|
|
54
|
+
});
|
|
55
|
+
}
|