@tpsdev-ai/flair 0.21.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -7
- package/SECURITY.md +24 -2
- package/config.yaml +32 -5
- package/dist/cli.js +1755 -213
- package/dist/deploy.js +3 -4
- package/dist/fleet-presence.js +98 -0
- package/dist/fleet-verify.js +291 -0
- package/dist/mcp-client-assertion.js +364 -0
- package/dist/probe.js +97 -0
- package/dist/rem/runner.js +82 -12
- package/dist/resources/AttentionQuery.js +356 -0
- package/dist/resources/Credential.js +11 -1
- package/dist/resources/Federation.js +23 -0
- package/dist/resources/MCPClientMetadata.js +88 -0
- package/dist/resources/Memory.js +52 -53
- package/dist/resources/MemoryBootstrap.js +422 -76
- package/dist/resources/MemoryReflect.js +105 -49
- package/dist/resources/MemoryUsage.js +104 -0
- package/dist/resources/OAuth.js +8 -1
- package/dist/resources/OrgEvent.js +11 -0
- package/dist/resources/Presence.js +218 -19
- package/dist/resources/RecordUsage.js +230 -0
- package/dist/resources/Relationship.js +98 -25
- package/dist/resources/SemanticSearch.js +85 -317
- package/dist/resources/SkillScan.js +15 -0
- package/dist/resources/WorkspaceState.js +11 -0
- package/dist/resources/agent-auth.js +60 -2
- package/dist/resources/auth-middleware.js +18 -9
- package/dist/resources/bm25.js +12 -6
- package/dist/resources/collision-lib.js +157 -0
- package/dist/resources/embeddings-boot.js +149 -0
- package/dist/resources/embeddings-provider.js +364 -106
- package/dist/resources/entity-vocab.js +139 -0
- package/dist/resources/health.js +97 -6
- package/dist/resources/mcp-client-metadata-fields.js +112 -0
- package/dist/resources/mcp-handler.js +1 -1
- package/dist/resources/mcp-tools.js +88 -10
- package/dist/resources/memory-reflect-lib.js +289 -0
- package/dist/resources/migration-boot.js +143 -0
- package/dist/resources/migrations/dir-safety.js +71 -0
- package/dist/resources/migrations/embedding-stamp.js +178 -0
- package/dist/resources/migrations/envelope.js +43 -0
- package/dist/resources/migrations/export.js +38 -0
- package/dist/resources/migrations/ledger.js +55 -0
- package/dist/resources/migrations/lock.js +154 -0
- package/dist/resources/migrations/progress.js +31 -0
- package/dist/resources/migrations/registry.js +36 -0
- package/dist/resources/migrations/risk-policy.js +23 -0
- package/dist/resources/migrations/runner.js +456 -0
- package/dist/resources/migrations/snapshot.js +127 -0
- package/dist/resources/migrations/source-fields.js +93 -0
- package/dist/resources/migrations/space.js +73 -0
- package/dist/resources/migrations/state.js +64 -0
- package/dist/resources/migrations/status.js +39 -0
- package/dist/resources/migrations/synthetic-test-migration.js +93 -0
- package/dist/resources/migrations/types.js +9 -0
- package/dist/resources/presence-internal.js +29 -0
- package/dist/resources/provenance.js +50 -0
- package/dist/resources/rate-limiter.js +8 -1
- package/dist/resources/scoring.js +124 -5
- package/dist/resources/semantic-retrieval-core.js +317 -0
- package/dist/version-handshake.js +122 -0
- package/package.json +4 -5
- package/schemas/event.graphql +5 -0
- package/schemas/memory.graphql +66 -0
- package/schemas/schema.graphql +5 -43
- package/schemas/workspace.graphql +3 -0
- package/dist/resources/IngestEvents.js +0 -162
- package/dist/resources/IssueTokens.js +0 -19
- package/dist/resources/ObsAgentSnapshot.js +0 -13
- package/dist/resources/ObsEventFeed.js +0 -13
- package/dist/resources/ObsOffice.js +0 -19
- package/dist/resources/ObservationCenter.js +0 -23
- package/ui/observation-center.html +0 -385
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* embedding-stamp.ts — the first registered migration (per the spec):
|
|
3
|
+
* re-embeds every Memory row whose `embeddingModel` doesn't match
|
|
4
|
+
* `getModelId()`. riskClass 'derived-only' — embeddings are recomputable
|
|
5
|
+
* from `content` (SOURCE_FIELDS), never the only copy of anything
|
|
6
|
+
* (invariant I), so this is the cheapest posture: metadata-only snapshot,
|
|
7
|
+
* no content-hash gate (row-count + stamp convergence only).
|
|
8
|
+
*
|
|
9
|
+
* THE GATE (`EMBEDDING_PREFIXES_ENABLED` in resources/embeddings-provider.ts)
|
|
10
|
+
* is now ON (flair#504, flipped and re-baselined through the ratchet gate),
|
|
11
|
+
* so `getModelId()` returns `<base>+searchprefix` — every row written before
|
|
12
|
+
* this flip was stamped with the bare base id, so it now reads as stale and
|
|
13
|
+
* gets picked up by this migration on the next boot that reaches it. This is
|
|
14
|
+
* this migration's first real payload: before the flip, `getModelId()`
|
|
15
|
+
* returned the bare base id unconditionally, so this migration — live and
|
|
16
|
+
* always-registered from the day it shipped — had nothing to detect. That
|
|
17
|
+
* was intentional groundwork, not dead code: proving the detect/re-embed
|
|
18
|
+
* mechanism end-to-end (this file, `test/integration/
|
|
19
|
+
* migrations-embedding-stamp-e2e.test.ts`) before it ever had real work to
|
|
20
|
+
* do was the point.
|
|
21
|
+
*
|
|
22
|
+
* Reuses Memory's OWN regen branch — never duplicates embedding logic —
|
|
23
|
+
* via the SAME mechanism `flair reembed` (src/cli.ts) already uses in
|
|
24
|
+
* production: a genuine `PUT /Memory/:id` HTTP request (admin-authenticated
|
|
25
|
+
* loopback), not an in-process call on `databases.flair.Memory` directly.
|
|
26
|
+
*
|
|
27
|
+
* THIS IS LOAD-BEARING, confirmed empirically while building
|
|
28
|
+
* test/integration/migrations-embedding-stamp-e2e.test.ts against real
|
|
29
|
+
* Harper: `databases.flair.Memory` (the property Harper exposes to OTHER
|
|
30
|
+
* modules, imported the same way resources/health.ts / MemoryReindex.ts /
|
|
31
|
+
* etc. already do for READS) resolves to the RAW underlying table, not the
|
|
32
|
+
* resources/Memory.ts SUBCLASS that carries the actual regen/dedup/auth
|
|
33
|
+
* logic — that subclass is only reachable via Harper's own REST dispatch
|
|
34
|
+
* for a genuine `/Memory` request. Calling `.put()` on the raw reference
|
|
35
|
+
* writes the record fine (confirmed: fields land exactly as passed) but
|
|
36
|
+
* silently skips every override in resources/Memory.ts, INCLUDING the
|
|
37
|
+
* `if (content.content && !content.embedding) { regenerate } ` branch this
|
|
38
|
+
* migration exists to trigger. Reads (`.search()`/`.get()`, used below for
|
|
39
|
+
* detect/countPending/candidate-selection) are unaffected by this — only
|
|
40
|
+
* `.put()` needs the real dispatch, hence the loopback HTTP call for
|
|
41
|
+
* exactly that one step.
|
|
42
|
+
*
|
|
43
|
+
* Query correctness (also verified against real Harper): Harper's
|
|
44
|
+
* `not_equal` comparator only matches rows where the attribute holds an
|
|
45
|
+
* EXPLICIT value (including explicit `null`) — a row where the attribute
|
|
46
|
+
* was NEVER SET AT ALL (`getIndexedValues()` returns `undefined` for a
|
|
47
|
+
* truly-absent property, never indexed regardless of `indexNulls`) is
|
|
48
|
+
* invisible to ANY condition-based query, not just `not_equal`. Clearing to
|
|
49
|
+
* `null` (never `undefined`) on write is therefore load-bearing: if the
|
|
50
|
+
* regen HTTP call fails (engine not yet warmed up, transient failure,
|
|
51
|
+
* admin credential unavailable this cycle), the row must land back in an
|
|
52
|
+
* EXPLICIT-null state — queryable and retried on the next boot — never a
|
|
53
|
+
* truly-absent one that would be permanently invisible to this migration
|
|
54
|
+
* again. The pending condition is an OR of `not_equal <current>` (catches a
|
|
55
|
+
* stale non-null model string) and `equals null` (catches that
|
|
56
|
+
* explicit-null state) — together they catch every state this migration's
|
|
57
|
+
* OWN writes can ever produce. A row whose `embeddingModel` was NEVER
|
|
58
|
+
* touched by anything (truly absent from its very first write — only
|
|
59
|
+
* possible if the embeddings engine was down for that entire write) is a
|
|
60
|
+
* known, narrow gap this bounded query cannot see;
|
|
61
|
+
* resources/migration-boot.ts mitigates the common case by waiting for the
|
|
62
|
+
* embeddings engine to settle before running migrations at all.
|
|
63
|
+
*/
|
|
64
|
+
import { databases } from "@harperfast/harper";
|
|
65
|
+
import { getModelId } from "../embeddings-provider.js";
|
|
66
|
+
function defaultMemoryTable() {
|
|
67
|
+
return databases.flair.Memory;
|
|
68
|
+
}
|
|
69
|
+
export const EMBEDDING_STAMP_ID = "embedding-stamp";
|
|
70
|
+
const REGEN_HTTP_TIMEOUT_MS = 20_000; // a real embedding compute can be slow on constrained hardware
|
|
71
|
+
/** Same admin-password resolution as resources/auth-middleware.ts's getAdminPass(). */
|
|
72
|
+
function resolveAdminAuthHeader() {
|
|
73
|
+
const pass = process.env.HDB_ADMIN_PASSWORD ?? process.env.FLAIR_ADMIN_PASSWORD;
|
|
74
|
+
if (!pass)
|
|
75
|
+
return null;
|
|
76
|
+
return "Basic " + Buffer.from(`admin:${pass}`).toString("base64");
|
|
77
|
+
}
|
|
78
|
+
/** Same HTTP_PORT env resolution src/cli.ts sets on every Harper spawn (see that file's grep for HTTP_PORT). */
|
|
79
|
+
function resolveSelfBaseUrl() {
|
|
80
|
+
const port = process.env.HTTP_PORT ?? "9926";
|
|
81
|
+
return `http://127.0.0.1:${port}`;
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Triggers Memory.put()'s regen branch via a genuine loopback HTTP PUT —
|
|
85
|
+
* the ONLY reliable way to reach resources/Memory.ts's subclass logic (see
|
|
86
|
+
* module doc). Returns true iff the request succeeded (2xx); NEVER throws
|
|
87
|
+
* — a failure just leaves the row in its current (queryable, explicit-null
|
|
88
|
+
* or stale-string) state for the next attempt.
|
|
89
|
+
*/
|
|
90
|
+
async function regenViaHttpPut(id, existing, fetchImpl) {
|
|
91
|
+
const authHeader = resolveAdminAuthHeader();
|
|
92
|
+
if (!authHeader)
|
|
93
|
+
return false; // no admin credential available this cycle — retried next boot
|
|
94
|
+
try {
|
|
95
|
+
const res = await fetchImpl(`${resolveSelfBaseUrl()}/Memory/${encodeURIComponent(id)}`, {
|
|
96
|
+
method: "PUT",
|
|
97
|
+
headers: { "Content-Type": "application/json", Authorization: authHeader },
|
|
98
|
+
body: JSON.stringify({ ...existing, embedding: null, embeddingModel: null }),
|
|
99
|
+
signal: AbortSignal.timeout(REGEN_HTTP_TIMEOUT_MS),
|
|
100
|
+
});
|
|
101
|
+
return res.ok;
|
|
102
|
+
}
|
|
103
|
+
catch {
|
|
104
|
+
return false;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* `getTable`/`getCurrentModelId`/`regen` are injectable so unit tests can
|
|
109
|
+
* exercise this migration's full detect/countPending/run logic against an
|
|
110
|
+
* in-memory fake table and a fake regen function (no real Harper, no real
|
|
111
|
+
* HTTP needed) — matching the mocking technique used throughout
|
|
112
|
+
* test/unit/*.ts (e.g. test/unit/instance-identity.test.ts). The real
|
|
113
|
+
* loopback-HTTP regen path is exercised for real in
|
|
114
|
+
* test/integration/migrations-embedding-stamp-e2e.test.ts.
|
|
115
|
+
*/
|
|
116
|
+
export function createEmbeddingStampMigration(getTable = defaultMemoryTable, getCurrentModelId = getModelId, regen = (id, existing) => regenViaHttpPut(id, existing, fetch)) {
|
|
117
|
+
function staleCondition() {
|
|
118
|
+
// OR-combined: `not_equal <current>` catches a stale non-null model
|
|
119
|
+
// string; `equals null` catches the explicit-null state this
|
|
120
|
+
// migration's own writes leave behind on a failed regen (see the
|
|
121
|
+
// module doc above — Harper's index never sees a TRULY ABSENT
|
|
122
|
+
// property, only an explicit null).
|
|
123
|
+
return [
|
|
124
|
+
{
|
|
125
|
+
operator: "or",
|
|
126
|
+
conditions: [
|
|
127
|
+
{ attribute: "embeddingModel", comparator: "not_equal", value: getCurrentModelId() },
|
|
128
|
+
{ attribute: "embeddingModel", comparator: "equals", value: null },
|
|
129
|
+
],
|
|
130
|
+
},
|
|
131
|
+
];
|
|
132
|
+
}
|
|
133
|
+
return {
|
|
134
|
+
id: EMBEDDING_STAMP_ID,
|
|
135
|
+
riskClass: "derived-only",
|
|
136
|
+
affectsTables: ["Memory"],
|
|
137
|
+
async detect() {
|
|
138
|
+
const table = getTable();
|
|
139
|
+
for await (const _row of table.search({ conditions: staleCondition(), limit: 1 })) {
|
|
140
|
+
return true;
|
|
141
|
+
}
|
|
142
|
+
return false;
|
|
143
|
+
},
|
|
144
|
+
async countPending() {
|
|
145
|
+
const table = getTable();
|
|
146
|
+
let n = 0;
|
|
147
|
+
for await (const _row of table.search({ conditions: staleCondition() }))
|
|
148
|
+
n++;
|
|
149
|
+
return n;
|
|
150
|
+
},
|
|
151
|
+
async run(batchSize) {
|
|
152
|
+
const table = getTable();
|
|
153
|
+
const current = getCurrentModelId();
|
|
154
|
+
const candidates = [];
|
|
155
|
+
for await (const row of table.search({ conditions: staleCondition(), limit: batchSize })) {
|
|
156
|
+
candidates.push(row);
|
|
157
|
+
}
|
|
158
|
+
const touchedIds = [];
|
|
159
|
+
for (const row of candidates) {
|
|
160
|
+
const id = String(row.id ?? "");
|
|
161
|
+
if (!id)
|
|
162
|
+
continue;
|
|
163
|
+
const existing = await table.get(id);
|
|
164
|
+
if (!existing)
|
|
165
|
+
continue; // deleted since the search above — nothing to fix
|
|
166
|
+
if (existing.embeddingModel === current)
|
|
167
|
+
continue; // already stamped by a concurrent runner — idempotent skip
|
|
168
|
+
const ok = await regen(id, existing);
|
|
169
|
+
if (ok)
|
|
170
|
+
touchedIds.push(id);
|
|
171
|
+
// A failed regen leaves the row untouched (still matching
|
|
172
|
+
// staleCondition — retried next batch/boot), never partially
|
|
173
|
+
// written or marked done.
|
|
174
|
+
}
|
|
175
|
+
return { processed: touchedIds.length, touchedIds };
|
|
176
|
+
},
|
|
177
|
+
};
|
|
178
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* envelope.ts — the corpus-wide integrity envelope: "full-corpus
|
|
3
|
+
* hash-of-source-field-hashes computed before first write and after
|
|
4
|
+
* completion; must match" (flair#695 invariant IV /
|
|
5
|
+
* Kern verdict), computed ONCE per boot cycle (K&S: "Envelope ASYNC after
|
|
6
|
+
* ready — boot serves immediately on the old shape; pre-hash runs async;
|
|
7
|
+
* migration deferred until it completes").
|
|
8
|
+
*
|
|
9
|
+
* `computeCorpusEnvelope` walks BOTH SOURCE_FIELDS tables (Memory,
|
|
10
|
+
* Relationship) once and returns both the single aggregate `corpusHash`
|
|
11
|
+
* (what schema-additive's full-envelope gate compares pre/post) AND the
|
|
12
|
+
* full `perRowHash` breakdown (`${table}:${id}` -> hash) — the latter isn't
|
|
13
|
+
* used by this module directly, but keeping the breakdown around lets a
|
|
14
|
+
* caller answer "did THIS specific row's source fields change" without a
|
|
15
|
+
* second full-corpus pass, if ever needed for diagnostics.
|
|
16
|
+
*
|
|
17
|
+
* Deliberately NOT scoped by risk class — this is the one integrity
|
|
18
|
+
* computation the whole cycle shares; risk-class-specific gate strictness
|
|
19
|
+
* (runner.ts) decides HOW MUCH of it each migration's completion gate
|
|
20
|
+
* actually checks.
|
|
21
|
+
*/
|
|
22
|
+
import { createHash } from "node:crypto";
|
|
23
|
+
import { hashSourceFields, sourceFieldsFor } from "./source-fields.js";
|
|
24
|
+
export const ENVELOPE_TABLES = ["Memory", "Relationship"];
|
|
25
|
+
export async function computeCorpusEnvelope(getTable, now) {
|
|
26
|
+
const perRowHash = new Map();
|
|
27
|
+
const parts = [];
|
|
28
|
+
for (const table of ENVELOPE_TABLES) {
|
|
29
|
+
const fields = sourceFieldsFor(table);
|
|
30
|
+
const accessor = getTable(table);
|
|
31
|
+
for await (const row of accessor.search({})) {
|
|
32
|
+
const id = String(row.id ?? "");
|
|
33
|
+
if (!id)
|
|
34
|
+
continue;
|
|
35
|
+
const h = hashSourceFields(row, fields);
|
|
36
|
+
perRowHash.set(`${table}:${id}`, h);
|
|
37
|
+
parts.push(`${table}:${id}:${h}`);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
parts.sort();
|
|
41
|
+
const corpusHash = createHash("sha256").update(parts.join("|")).digest("hex");
|
|
42
|
+
return { perRowHash, corpusHash, computedAt: now().toISOString() };
|
|
43
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* export.ts — content-only logical export fallback (ladder step 4):
|
|
3
|
+
* "Content-only logical export fallback: embeddings/indexes dominate
|
|
4
|
+
* data-dir size; the irreplaceable part (memory content + provenance)
|
|
5
|
+
* exports far smaller. If the physical snapshot doesn't fit but the content
|
|
6
|
+
* export does → export content, proceed (derived is recomputable by
|
|
7
|
+
* definition)." (flair#695, space-pressure step 4)
|
|
8
|
+
*
|
|
9
|
+
* Exports SOURCE_FIELDS ONLY (+id) — never derived fields (embedding is the
|
|
10
|
+
* dominant size driver this exists to skip) — as JSONL, one row per line,
|
|
11
|
+
* under the SAME 0700/0600 discipline as snapshot.ts.
|
|
12
|
+
*/
|
|
13
|
+
import { statSync } from "node:fs";
|
|
14
|
+
import { join } from "node:path";
|
|
15
|
+
import { ensureSecureDir, writeSecureFile } from "./dir-safety.js";
|
|
16
|
+
import { sourceFieldsFor } from "./source-fields.js";
|
|
17
|
+
function sanitizeIdPart(s) {
|
|
18
|
+
return s.replace(/[^a-zA-Z0-9_-]/g, "_");
|
|
19
|
+
}
|
|
20
|
+
export function createContentOnlyExport(opts, deps) {
|
|
21
|
+
const now = deps.now();
|
|
22
|
+
const iso = now.toISOString().replace(/[:.]/g, "-");
|
|
23
|
+
const dir = join(deps.exportRoot, `${sanitizeIdPart(opts.migrationId)}-${opts.table}-${iso}`);
|
|
24
|
+
ensureSecureDir(dir);
|
|
25
|
+
const fields = sourceFieldsFor(opts.table);
|
|
26
|
+
const lines = opts.rows.map((row) => {
|
|
27
|
+
const picked = { id: row.id };
|
|
28
|
+
for (const f of fields)
|
|
29
|
+
picked[f] = row[f] ?? null;
|
|
30
|
+
return JSON.stringify(picked);
|
|
31
|
+
});
|
|
32
|
+
const body = lines.join("\n") + (lines.length ? "\n" : "");
|
|
33
|
+
const path = join(dir, `${opts.table}.jsonl`);
|
|
34
|
+
writeSecureFile(path, body, dir);
|
|
35
|
+
const metaPath = join(dir, "manifest.json");
|
|
36
|
+
writeSecureFile(metaPath, JSON.stringify({ migrationId: opts.migrationId, table: opts.table, fromVersion: opts.fromVersion, createdAt: now.toISOString(), rowCount: opts.rows.length }, null, 2) + "\n", dir);
|
|
37
|
+
return { dir, path, bytes: statSync(path).size, rowCount: opts.rows.length };
|
|
38
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ledger.ts — the migration ledger OrgEvent (invariant IV: "every migration
|
|
3
|
+
* records an OrgEvent (initiator, versions, scope, outcome, duration) —
|
|
4
|
+
* migrations get decision provenance. Dogfood.")
|
|
5
|
+
*
|
|
6
|
+
* Sherlock verdict: "Ledger discloses structural metadata only — versions,
|
|
7
|
+
* counts, outcome, duration; never memory IDs or content summaries." The
|
|
8
|
+
* shape below is EXACTLY the Kern-verdict field list (migration id,
|
|
9
|
+
* initiator, from/to version, scope, start/end, outcome, rows
|
|
10
|
+
* processed/remaining, error if halted, hash envelope) — nothing else is
|
|
11
|
+
* ever written into `detail`.
|
|
12
|
+
*
|
|
13
|
+
* Written via an internal (no-HTTP-context) call to
|
|
14
|
+
* `databases.flair.OrgEvent.put()` — resolveAgentAuth(undefined) resolves
|
|
15
|
+
* to `{kind: "internal"}` (trusted; see resources/agent-auth.ts), the same
|
|
16
|
+
* pattern resources/auth-middleware.ts's backfillEmbedding() and
|
|
17
|
+
* resources/MemoryReindex.ts's `_reindex` path already rely on for
|
|
18
|
+
* server-internal writes with no HTTP caller behind them.
|
|
19
|
+
*/
|
|
20
|
+
import { databases } from "@harperfast/harper";
|
|
21
|
+
function defaultOrgEventTable() {
|
|
22
|
+
return databases.flair.OrgEvent;
|
|
23
|
+
}
|
|
24
|
+
/** Structural-only detail blob — see the module doc; never memory IDs/content. */
|
|
25
|
+
export function buildLedgerDetail(evt) {
|
|
26
|
+
return JSON.stringify({
|
|
27
|
+
migrationId: evt.migrationId,
|
|
28
|
+
initiator: evt.initiator,
|
|
29
|
+
fromVersion: evt.fromVersion,
|
|
30
|
+
toVersion: evt.toVersion,
|
|
31
|
+
scope: evt.scope,
|
|
32
|
+
startedAt: evt.startedAt,
|
|
33
|
+
endedAt: evt.endedAt,
|
|
34
|
+
outcome: evt.outcome,
|
|
35
|
+
rowsProcessed: evt.rowsProcessed,
|
|
36
|
+
rowsRemaining: evt.rowsRemaining,
|
|
37
|
+
hashEnvelopeMatch: evt.hashEnvelopeMatch,
|
|
38
|
+
...(evt.error ? { error: evt.error } : {}),
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
export async function writeLedgerEvent(evt, deps = {}) {
|
|
42
|
+
const table = deps.orgEventTable ?? defaultOrgEventTable();
|
|
43
|
+
const id = `migration-${evt.migrationId}-${evt.endedAt}`;
|
|
44
|
+
const remainingNote = evt.rowsRemaining > 0 ? `, ${evt.rowsRemaining} remaining` : "";
|
|
45
|
+
await table.put({
|
|
46
|
+
id,
|
|
47
|
+
authorId: "flair-migrations",
|
|
48
|
+
kind: "migration",
|
|
49
|
+
scope: evt.scope,
|
|
50
|
+
summary: `migration ${evt.migrationId} ${evt.outcome} (${evt.rowsProcessed} row${evt.rowsProcessed === 1 ? "" : "s"} processed${remainingNote})`,
|
|
51
|
+
detail: buildLedgerDetail(evt),
|
|
52
|
+
refId: evt.migrationId,
|
|
53
|
+
createdAt: evt.endedAt,
|
|
54
|
+
});
|
|
55
|
+
}
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* lock.ts — single-flight guard (Kern verdict, 2026-07-12):
|
|
3
|
+
* "Single-flight v1 = in-process mutex + stale-tolerant file lock (flock on
|
|
4
|
+
* well-known path; dead-holder → break and proceed). Fabric-wide ordering =
|
|
5
|
+
* follow-on with its own review, NOT this slice."
|
|
6
|
+
*
|
|
7
|
+
* Two layers:
|
|
8
|
+
* 1. An in-process mutex (module-scope flag) — cheapest guard against two
|
|
9
|
+
* concurrent async callers inside the SAME Node process (e.g. a
|
|
10
|
+
* duplicate boot-trigger firing twice).
|
|
11
|
+
* 2. A file lock at a well-known path under the data dir. Node has no
|
|
12
|
+
* portable flock() binding, so this uses exclusive file CREATE
|
|
13
|
+
* (O_CREAT|O_EXCL — atomic at the OS level, the same primitive flock
|
|
14
|
+
* ultimately reduces to for a single-host guard) instead. The lock file
|
|
15
|
+
* holds `{pid, hostname, startedAt}`. A lock is considered STALE (and
|
|
16
|
+
* is broken/removed before the new attempt) when either: the recorded
|
|
17
|
+
* pid is no longer alive on this host, OR the file is older than
|
|
18
|
+
* `staleMs` (covers a cross-host holder this process can't liveness-
|
|
19
|
+
* check by pid — Fabric-wide coordination is explicitly out of scope
|
|
20
|
+
* for this slice, so age is the only signal available for that case).
|
|
21
|
+
*
|
|
22
|
+
* This is a SINGLE-HOST guard, not a Fabric-wide one — multiple Flair
|
|
23
|
+
* instances in a Fabric cluster each have their OWN data dir today, so this
|
|
24
|
+
* is sufficient for v1. Fabric-wide ordering (one lock across a cluster
|
|
25
|
+
* sharing storage) is explicitly deferred, per the verdict above.
|
|
26
|
+
*/
|
|
27
|
+
import { existsSync, mkdirSync, openSync, closeSync, writeSync, readFileSync, unlinkSync, statSync, utimesSync, constants as fsConstants } from "node:fs";
|
|
28
|
+
import { dirname } from "node:path";
|
|
29
|
+
import { hostname as osHostname } from "node:os";
|
|
30
|
+
export const DEFAULT_STALE_MS = 5 * 60 * 1000; // 5 minutes with no heartbeat touch = stale
|
|
31
|
+
export function defaultIsProcessAlive(pid) {
|
|
32
|
+
try {
|
|
33
|
+
process.kill(pid, 0);
|
|
34
|
+
return true;
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
return false;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
export function defaultLockDeps(lockPath) {
|
|
41
|
+
return {
|
|
42
|
+
lockPath,
|
|
43
|
+
staleMs: DEFAULT_STALE_MS,
|
|
44
|
+
isProcessAlive: defaultIsProcessAlive,
|
|
45
|
+
now: () => Date.now(),
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
// In-process mutex — module-scope, so it's shared by every caller within
|
|
49
|
+
// this one Node process regardless of which lockPath they pass (a single
|
|
50
|
+
// process only ever wants to run one migration cycle at a time).
|
|
51
|
+
let inProcessHeld = false;
|
|
52
|
+
/** Test-only reset for the in-process mutex (mirrors instance-identity.ts's `_reset*ForTests` idiom). */
|
|
53
|
+
export function _resetInProcessLockForTests() {
|
|
54
|
+
inProcessHeld = false;
|
|
55
|
+
}
|
|
56
|
+
function readHolder(lockPath) {
|
|
57
|
+
try {
|
|
58
|
+
const raw = JSON.parse(readFileSync(lockPath, "utf-8"));
|
|
59
|
+
if (typeof raw?.pid === "number" && typeof raw?.hostname === "string" && typeof raw?.startedAt === "string") {
|
|
60
|
+
return raw;
|
|
61
|
+
}
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Attempts to acquire the migration lock. On success, returns `release()`
|
|
70
|
+
* (removes the lock file + clears the in-process flag) and `touch()`
|
|
71
|
+
* (updates the lock file's mtime — the runner calls this periodically
|
|
72
|
+
* during a long-running migration so ANOTHER process's staleness check
|
|
73
|
+
* doesn't mistake a genuinely-still-running holder for dead).
|
|
74
|
+
*/
|
|
75
|
+
export function acquireMigrationLock(deps) {
|
|
76
|
+
const resolved = { ...defaultLockDeps(deps.lockPath), ...deps };
|
|
77
|
+
const { lockPath, staleMs, isProcessAlive, now } = resolved;
|
|
78
|
+
if (inProcessHeld) {
|
|
79
|
+
return { acquired: false, reason: "already held in-process (another async caller in this process holds it)" };
|
|
80
|
+
}
|
|
81
|
+
mkdirSync(dirname(lockPath), { recursive: true, mode: 0o700 });
|
|
82
|
+
if (existsSync(lockPath)) {
|
|
83
|
+
const holder = readHolder(lockPath);
|
|
84
|
+
let ageMs = Infinity;
|
|
85
|
+
try {
|
|
86
|
+
ageMs = now() - statSync(lockPath).mtimeMs;
|
|
87
|
+
}
|
|
88
|
+
catch {
|
|
89
|
+
/* vanished between existsSync and stat — fall through to acquire */
|
|
90
|
+
}
|
|
91
|
+
const dead = !holder || !isProcessAlive(holder.pid) || ageMs > staleMs;
|
|
92
|
+
if (dead) {
|
|
93
|
+
try {
|
|
94
|
+
unlinkSync(lockPath);
|
|
95
|
+
}
|
|
96
|
+
catch {
|
|
97
|
+
/* another racer already broke it — fine, continue to the exclusive create below */
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
else {
|
|
101
|
+
return {
|
|
102
|
+
acquired: false,
|
|
103
|
+
reason: `held by pid ${holder.pid} on ${holder.hostname} since ${holder.startedAt}`,
|
|
104
|
+
holder,
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
let fd;
|
|
109
|
+
try {
|
|
110
|
+
// O_EXCL: atomic exclusive create — fails with EEXIST if another
|
|
111
|
+
// process/racer won the create between our stale-check above and here.
|
|
112
|
+
fd = openSync(lockPath, fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_WRONLY, 0o600);
|
|
113
|
+
}
|
|
114
|
+
catch (err) {
|
|
115
|
+
const code = err?.code;
|
|
116
|
+
if (code === "EEXIST") {
|
|
117
|
+
return { acquired: false, reason: "lost the race to acquire the lock file" };
|
|
118
|
+
}
|
|
119
|
+
throw err;
|
|
120
|
+
}
|
|
121
|
+
try {
|
|
122
|
+
const info = { pid: process.pid, hostname: osHostname(), startedAt: new Date(now()).toISOString() };
|
|
123
|
+
writeSync(fd, JSON.stringify(info));
|
|
124
|
+
}
|
|
125
|
+
finally {
|
|
126
|
+
closeSync(fd);
|
|
127
|
+
}
|
|
128
|
+
inProcessHeld = true;
|
|
129
|
+
let released = false;
|
|
130
|
+
return {
|
|
131
|
+
acquired: true,
|
|
132
|
+
release: () => {
|
|
133
|
+
if (released)
|
|
134
|
+
return;
|
|
135
|
+
released = true;
|
|
136
|
+
inProcessHeld = false;
|
|
137
|
+
try {
|
|
138
|
+
unlinkSync(lockPath);
|
|
139
|
+
}
|
|
140
|
+
catch {
|
|
141
|
+
/* already gone — fine */
|
|
142
|
+
}
|
|
143
|
+
},
|
|
144
|
+
touch: () => {
|
|
145
|
+
const t = new Date(now());
|
|
146
|
+
try {
|
|
147
|
+
utimesSync(lockPath, t, t);
|
|
148
|
+
}
|
|
149
|
+
catch {
|
|
150
|
+
/* best-effort heartbeat — a failed touch must never break the migration */
|
|
151
|
+
}
|
|
152
|
+
},
|
|
153
|
+
};
|
|
154
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
let cycleStatus = { phase: "idle" };
|
|
2
|
+
const progressById = new Map();
|
|
3
|
+
export function setCyclePhase(phase, error) {
|
|
4
|
+
cycleStatus = { phase, lastCycleError: error, lastCycleAt: new Date().toISOString() };
|
|
5
|
+
}
|
|
6
|
+
export function getCycleStatus() {
|
|
7
|
+
return { ...cycleStatus };
|
|
8
|
+
}
|
|
9
|
+
export function setMigrationProgress(p) {
|
|
10
|
+
progressById.set(p.id, p);
|
|
11
|
+
}
|
|
12
|
+
export function getMigrationProgress(id) {
|
|
13
|
+
return progressById.get(id);
|
|
14
|
+
}
|
|
15
|
+
export function listMigrationProgress() {
|
|
16
|
+
return Array.from(progressById.values());
|
|
17
|
+
}
|
|
18
|
+
/** Seeds an "idle" entry for every registered migration id — called once at
|
|
19
|
+
* boot-trigger registration time, synchronously, so `health` always has a
|
|
20
|
+
* row per registered migration even before the async cycle has run at all. */
|
|
21
|
+
export function seedIdleProgress(ids) {
|
|
22
|
+
for (const id of ids) {
|
|
23
|
+
if (!progressById.has(id)) {
|
|
24
|
+
progressById.set(id, { id, rowsDone: 0, rowsRemaining: 0, state: "idle" });
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
export function _resetProgressForTests() {
|
|
29
|
+
progressById.clear();
|
|
30
|
+
cycleStatus = { phase: "idle" };
|
|
31
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* registry.ts — the MigrationRegistry: an ordered list of registered
|
|
3
|
+
* migrations. embedding-stamp always registers; the synthetic CI-only
|
|
4
|
+
* variant registers ONLY when shouldRegisterSyntheticMigration() is true
|
|
5
|
+
* (see synthetic-test-migration.ts's doc for the exact gating rule).
|
|
6
|
+
*/
|
|
7
|
+
import { createEmbeddingStampMigration } from "./embedding-stamp.js";
|
|
8
|
+
import { createSyntheticTestMigration, shouldRegisterSyntheticMigration } from "./synthetic-test-migration.js";
|
|
9
|
+
export class MigrationRegistry {
|
|
10
|
+
migrations = [];
|
|
11
|
+
register(migration) {
|
|
12
|
+
if (this.migrations.some((m) => m.id === migration.id)) {
|
|
13
|
+
throw new Error(`migration id already registered: ${migration.id}`);
|
|
14
|
+
}
|
|
15
|
+
this.migrations.push(migration);
|
|
16
|
+
return this;
|
|
17
|
+
}
|
|
18
|
+
list() {
|
|
19
|
+
return this.migrations;
|
|
20
|
+
}
|
|
21
|
+
get(id) {
|
|
22
|
+
return this.migrations.find((m) => m.id === id);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Builds the production registry. Always includes embedding-stamp;
|
|
27
|
+
* conditionally includes the CI-only synthetic migration.
|
|
28
|
+
*/
|
|
29
|
+
export function buildRegistry(env = process.env) {
|
|
30
|
+
const registry = new MigrationRegistry();
|
|
31
|
+
registry.register(createEmbeddingStampMigration());
|
|
32
|
+
if (shouldRegisterSyntheticMigration(env)) {
|
|
33
|
+
registry.register(createSyntheticTestMigration());
|
|
34
|
+
}
|
|
35
|
+
return registry;
|
|
36
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
const POSTURE = {
|
|
2
|
+
"derived-only": {
|
|
3
|
+
riskClass: "derived-only",
|
|
4
|
+
batchSize: 50,
|
|
5
|
+
snapshotScope: "metadata-only",
|
|
6
|
+
gate: "count+marker",
|
|
7
|
+
},
|
|
8
|
+
"schema-additive": {
|
|
9
|
+
riskClass: "schema-additive",
|
|
10
|
+
batchSize: 50,
|
|
11
|
+
snapshotScope: "schema+metadata",
|
|
12
|
+
gate: "count+full-envelope",
|
|
13
|
+
},
|
|
14
|
+
"content-transform": {
|
|
15
|
+
riskClass: "content-transform",
|
|
16
|
+
batchSize: 10,
|
|
17
|
+
snapshotScope: "pointers+metadata",
|
|
18
|
+
gate: "count+old-row-envelope+new-row-presence",
|
|
19
|
+
},
|
|
20
|
+
};
|
|
21
|
+
export function postureFor(riskClass) {
|
|
22
|
+
return POSTURE[riskClass];
|
|
23
|
+
}
|