@tpsdev-ai/flair 0.53.0 → 0.54.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -1
- package/dist/build-info.json +3 -3
- package/dist/cli.js +1791 -15648
- package/dist/commands/agent.js +453 -0
- package/dist/commands/attention.js +121 -0
- package/dist/commands/backup.js +115 -0
- package/dist/commands/bootstrap.js +91 -0
- package/dist/commands/bridge.js +608 -0
- package/dist/commands/deploy.js +180 -0
- package/dist/commands/doctor.js +1654 -0
- package/dist/commands/export.js +110 -0
- package/dist/commands/federation.js +1575 -0
- package/dist/commands/fleet.js +73 -0
- package/dist/commands/grant.js +109 -0
- package/dist/commands/hook.js +193 -0
- package/dist/commands/idp.js +193 -0
- package/dist/commands/import.js +134 -0
- package/dist/commands/init.js +1203 -0
- package/dist/commands/inspect.js +45 -0
- package/dist/commands/keys.js +187 -0
- package/dist/commands/mcp.js +707 -0
- package/dist/commands/memory.js +501 -0
- package/dist/commands/migrate-harness-memory.js +270 -0
- package/dist/commands/orgevent.js +138 -0
- package/dist/commands/presence.js +76 -0
- package/dist/commands/principal.js +338 -0
- package/dist/commands/quality.js +1164 -0
- package/dist/commands/reembed.js +296 -0
- package/dist/commands/relationship.js +76 -0
- package/dist/commands/rem.js +1048 -0
- package/dist/commands/restore.js +130 -0
- package/dist/commands/search.js +244 -0
- package/dist/commands/service.js +315 -0
- package/dist/commands/session.js +184 -0
- package/dist/commands/soul.js +155 -0
- package/dist/commands/status.js +914 -0
- package/dist/commands/test.js +93 -0
- package/dist/commands/uninstall.js +143 -0
- package/dist/commands/upgrade.js +1592 -0
- package/dist/commands/workspace.js +114 -0
- package/dist/deploy.js +24 -0
- package/dist/fabric-npm-install.js +87 -0
- package/dist/federation-verify.js +498 -0
- package/dist/fleet-verify.js +144 -21
- package/dist/install/clients.js +167 -0
- package/dist/lib/auth-resolve.js +76 -1
- package/dist/lib/daemon-liveness.js +131 -2
- package/dist/lib/doctor-config-path.js +61 -0
- package/dist/lib/doctor-federation-driver.js +189 -0
- package/dist/lib/doctor-run.js +40 -0
- package/dist/lib/entity-vocab-cli.js +3 -3
- package/dist/lib/federation-pair-identity.js +47 -0
- package/dist/lib/launchd-repair.js +5 -4
- package/dist/lib/ops-api-bind.js +115 -0
- package/dist/lib/owned-pins.js +219 -0
- package/dist/lib/uninstall-purge.js +218 -0
- package/dist/rem/restore.js +8 -10
- package/dist/resources/AgentReadPosition.js +74 -0
- package/dist/resources/Federation.js +8 -2
- package/dist/resources/Memory.js +4 -3
- package/dist/resources/MemoryBootstrap.js +41 -25
- package/dist/resources/MemoryCandidate.js +5 -6
- package/dist/resources/OrgEventCatchup.js +126 -47
- package/dist/resources/agent-read-position-lib.js +83 -0
- package/dist/resources/agent-read-position.js +120 -0
- package/dist/resources/embeddings-boot.js +32 -0
- package/dist/resources/federation-peer-liveness.js +73 -0
- package/dist/resources/health.js +68 -19
- package/dist/resources/mcp-tools.js +43 -279
- package/dist/resources/memory-visibility.js +3 -3
- package/dist/resources/migration-boot.js +59 -18
- package/dist/resources/migrations/embedding-stamp.js +20 -1
- package/dist/resources/migrations/recheck.js +43 -0
- package/dist/resources/migrations/runner.js +6 -1
- package/dist/resources/migrations/stamp-outstanding.js +171 -0
- package/dist/resources/migrations/visibility-backfill.js +2 -2
- package/dist/resources/org-event-catchup-lib.js +47 -0
- package/dist/resources/record-owner-guard.js +1 -0
- package/dist/stamp-migration-verify.js +163 -0
- package/dist/stamp-outstanding.js +144 -0
- package/docs/api-reference.md +4 -2
- package/docs/deploying-on-fabric.md +11 -10
- package/docs/deployment.md +3 -1
- package/docs/federation.md +19 -0
- package/docs/hosted-on-fabric.md +3 -3
- package/docs/quickstart.md +2 -1
- package/docs/releasing.md +15 -7
- package/docs/spoke-bringup.md +10 -5
- package/docs/standalone-local.md +3 -1
- package/docs/upgrade.md +25 -6
- package/node_modules/@tpsdev-ai/flair-tool-descriptors/LICENSE +19 -0
- package/node_modules/@tpsdev-ai/flair-tool-descriptors/README.md +22 -0
- package/node_modules/@tpsdev-ai/flair-tool-descriptors/dist/index.d.ts +70 -0
- package/node_modules/@tpsdev-ai/flair-tool-descriptors/dist/index.js +665 -0
- package/node_modules/@tpsdev-ai/flair-tool-descriptors/package.json +46 -0
- package/package.json +9 -4
- package/schemas/agent.graphql +15 -0
|
@@ -19,6 +19,14 @@
|
|
|
19
19
|
* migrations-embedding-stamp-e2e.test.ts`) before it ever had real work to
|
|
20
20
|
* do was the point.
|
|
21
21
|
*
|
|
22
|
+
* flair#1073: that self-heal sat idle for days on a Fabric instance. The
|
|
23
|
+
* runner is boot-keyed; a first detect() against a not-yet-visible corpus
|
|
24
|
+
* (or a version-keyed short-circuit after a no-op complete) marked this
|
|
25
|
+
* migration done while 554 pre-flip rows stayed stale. `alwaysDetect`
|
|
26
|
+
* plus delayed follow-up cycles close that; /HealthDetail names this
|
|
27
|
+
* migration when the corpus is still split. run() has no batch cap — the
|
|
28
|
+
* runner loops until processed=0 — and a re-run is idempotent.
|
|
29
|
+
*
|
|
22
30
|
* Reuses Memory's OWN regen branch — never duplicates embedding logic —
|
|
23
31
|
* via the SAME mechanism `flair reembed` (src/cli.ts) already uses in
|
|
24
32
|
* production: a genuine `PUT /Memory/:id` HTTP request (admin-authenticated
|
|
@@ -128,10 +136,11 @@
|
|
|
128
136
|
import { databases } from "harper";
|
|
129
137
|
import { getModelId } from "../embeddings-provider.js";
|
|
130
138
|
import { currentSpaceRawForms, isCurrentSpaceStamp } from "../embedding-space-guard.js";
|
|
139
|
+
import { EMBEDDING_STAMP_ID } from "./stamp-outstanding.js";
|
|
140
|
+
export { EMBEDDING_STAMP_ID };
|
|
131
141
|
function defaultMemoryTable() {
|
|
132
142
|
return databases.flair.Memory;
|
|
133
143
|
}
|
|
134
|
-
export const EMBEDDING_STAMP_ID = "embedding-stamp";
|
|
135
144
|
const REGEN_HTTP_TIMEOUT_MS = 20_000; // a real embedding compute can be slow on constrained hardware
|
|
136
145
|
/** Same admin-password resolution as resources/auth-middleware.ts's getAdminPass(). */
|
|
137
146
|
function resolveAdminAuthHeader() {
|
|
@@ -224,6 +233,16 @@ export function createEmbeddingStampMigration(getTable = defaultMemoryTable, get
|
|
|
224
233
|
id: EMBEDDING_STAMP_ID,
|
|
225
234
|
riskClass: "derived-only",
|
|
226
235
|
affectsTables: ["Memory"],
|
|
236
|
+
// flair#1073: pending work is "does any row's stamp differ from
|
|
237
|
+
// getModelId()", not "did this version already record success." A
|
|
238
|
+
// Fabric boot that detect()'d empty (tables ready, corpus not yet
|
|
239
|
+
// visible) or a no-op complete at the searchprefix-flip version wrote
|
|
240
|
+
// success and then skipped every later boot at that version — 554
|
|
241
|
+
// pre-flip rows sat stale for days. alwaysDetect keeps the cheap
|
|
242
|
+
// limit=1 corpus read on every cycle so a false complete cannot hide
|
|
243
|
+
// remaining work. run() is idempotent; a re-run over already-current
|
|
244
|
+
// rows is a no-op.
|
|
245
|
+
alwaysDetect: true,
|
|
227
246
|
async detect() {
|
|
228
247
|
const table = getTable();
|
|
229
248
|
for await (const _row of table.search({ conditions: staleCondition(), limit: 1 })) {
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* recheck.ts — follow-up migration cycles after the boot-keyed first pass
|
|
3
|
+
* (flair#1073).
|
|
4
|
+
*
|
|
5
|
+
* The boot runner fires once via `setImmediate` after tables exist. On a
|
|
6
|
+
* Fabric component deploy that is not enough: Memory.search can be a live
|
|
7
|
+
* accessor before the replica's rows are visible, so embedding-stamp's
|
|
8
|
+
* detect() returns false, the cycle records "nothing pending" / completed,
|
|
9
|
+
* and a long-lived process never looks again. These delayed rechecks are
|
|
10
|
+
* the live self-heal — the same runMigrationCycle, single-flight lock
|
|
11
|
+
* included, so a no-op is cheap (alwaysDetect detect() is limit=1) and a
|
|
12
|
+
* mid-flight first cycle just gets `single-flight` from the later one.
|
|
13
|
+
*
|
|
14
|
+
* Delays are spaced to cover the Fabric boot race (tens of seconds) and a
|
|
15
|
+
* slow replica catch-up (minutes), not an infinite poll. A process that
|
|
16
|
+
* is still split after the last delay will be picked up on the next
|
|
17
|
+
* restart (alwaysDetect) or the next `upgrade --target` stamp-convergence
|
|
18
|
+
* verify.
|
|
19
|
+
*/
|
|
20
|
+
export const DEFAULT_STAMP_RECHECK_DELAYS_MS = [
|
|
21
|
+
30_000,
|
|
22
|
+
120_000,
|
|
23
|
+
600_000,
|
|
24
|
+
];
|
|
25
|
+
export function scheduleFollowUpCycles(opts) {
|
|
26
|
+
const delaysMs = opts.delaysMs ?? DEFAULT_STAMP_RECHECK_DELAYS_MS;
|
|
27
|
+
const setT = opts.setTimeoutFn ?? setTimeout;
|
|
28
|
+
const clearT = opts.clearTimeoutFn ?? clearTimeout;
|
|
29
|
+
const timers = [];
|
|
30
|
+
for (const ms of delaysMs) {
|
|
31
|
+
timers.push(setT(() => {
|
|
32
|
+
void opts.run();
|
|
33
|
+
}, ms));
|
|
34
|
+
}
|
|
35
|
+
return {
|
|
36
|
+
delaysMs,
|
|
37
|
+
cancel() {
|
|
38
|
+
for (const t of timers)
|
|
39
|
+
clearT(t);
|
|
40
|
+
timers.length = 0;
|
|
41
|
+
},
|
|
42
|
+
};
|
|
43
|
+
}
|
|
@@ -150,7 +150,7 @@ async function runCycleLocked(deps, lock) {
|
|
|
150
150
|
const state = readMigrationState(deps.statePath);
|
|
151
151
|
const candidates = [];
|
|
152
152
|
for (const migration of deps.registry.list()) {
|
|
153
|
-
if (isShortCircuited(state, migration.id, deps.runningVersion)) {
|
|
153
|
+
if (isShortCircuited(state, migration.id, deps.runningVersion) && !migration.alwaysDetect) {
|
|
154
154
|
// flair#812: a short-circuit is an ASSERTION READ OFF A FILE, not an
|
|
155
155
|
// observation of the corpus — and that file is hand-editable (the
|
|
156
156
|
// documented remediation for a stuck migration is to correct it by
|
|
@@ -159,6 +159,11 @@ async function runCycleLocked(deps, lock) {
|
|
|
159
159
|
// The skip itself is unchanged (it is the documented cheap path); it
|
|
160
160
|
// now just says what it is, so `flair doctor` never presents an
|
|
161
161
|
// unverified claim as a verified one.
|
|
162
|
+
//
|
|
163
|
+
// flair#1073: `alwaysDetect` migrations skip this path — their
|
|
164
|
+
// pending work is a corpus property, so a version-keyed "already
|
|
165
|
+
// done" marker can lie (embedding-stamp completed at 0.30.0 while
|
|
166
|
+
// 554 pre-flip rows were still stale). detect() is still cheap.
|
|
162
167
|
setMigrationProgress({
|
|
163
168
|
id: migration.id,
|
|
164
169
|
rowsDone: 0,
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* stamp-outstanding.ts — corpus-derived "is embedding-stamp still pending?"
|
|
3
|
+
* (flair#1073). Harper-free, no embedding-space-guard import: that module
|
|
4
|
+
* schedules a boot pre-warm at load. This file is the Harper/HealthDetail
|
|
5
|
+
* canonical. The CLI ships a copy at src/stamp-outstanding.ts because
|
|
6
|
+
* tsconfig.cli.json's rootDir is `src` and the published tree cannot
|
|
7
|
+
* resolve `../resources/...` from dist/cli.js. Stay in sync via
|
|
8
|
+
* test/unit/stamp-outstanding-cli-parity.test.ts.
|
|
9
|
+
*
|
|
10
|
+
* THE point of this module: runner bookkeeping (`state: completed`,
|
|
11
|
+
* `rowsDone: 0`) is not the signal. A Fabric boot that detect()'d empty,
|
|
12
|
+
* or a version-keyed short-circuit, can mark embedding-stamp done while
|
|
13
|
+
* the corpus is still split. Outstanding is derived from modelCounts vs
|
|
14
|
+
* the current space, then annotated with whatever the runner last said
|
|
15
|
+
* so the warning can name the migration AND the consequences (search
|
|
16
|
+
* unreliable, dedup inactive) instead of a generic mixed-models symptom
|
|
17
|
+
* with a manual `flair reembed` remedy.
|
|
18
|
+
*/
|
|
19
|
+
/** Stable id — kept here so Harper resources can import this file without
|
|
20
|
+
* loading embedding-stamp.ts (which statically imports `harper`).
|
|
21
|
+
* Re-exported from embedding-stamp.ts so existing imports keep working.
|
|
22
|
+
* The CLI copy at src/stamp-outstanding.ts exports the same constant. */
|
|
23
|
+
export const EMBEDDING_STAMP_ID = "embedding-stamp";
|
|
24
|
+
/** Mirrors embedding-space-guard's normalizeStamp / NON_SPACE_STAMPS —
|
|
25
|
+
* inlined so this file never loads that module's boot side effect. */
|
|
26
|
+
const NON_SPACE_STAMPS = new Set(["hash-512d"]);
|
|
27
|
+
const DEFAULT_ENGINE = "gguf";
|
|
28
|
+
export function normalizeStampForOutstanding(stamp) {
|
|
29
|
+
if (stamp == null)
|
|
30
|
+
return null;
|
|
31
|
+
const s = String(stamp).trim();
|
|
32
|
+
if (s === "" || NON_SPACE_STAMPS.has(s))
|
|
33
|
+
return null;
|
|
34
|
+
return s.includes(":") ? s : `${DEFAULT_ENGINE}:${s}`;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Current-space id for a client that only has `modelCounts` (no live
|
|
38
|
+
* `getModelId()`). Prefers an existing `+searchprefix` stamp so a healthy
|
|
39
|
+
* bare+`gguf:` pair of the SAME space does not invent a foreign current.
|
|
40
|
+
* A uniformly pre-flip corpus has no suffix — append `+searchprefix` so
|
|
41
|
+
* those rows read as stale (the HealthDetail / quality-report gap Bugbot
|
|
42
|
+
* flagged on flair#1606).
|
|
43
|
+
*/
|
|
44
|
+
export function resolveCurrentModelId(modelCounts, explicit) {
|
|
45
|
+
const trimmed = explicit?.trim();
|
|
46
|
+
if (trimmed)
|
|
47
|
+
return trimmed;
|
|
48
|
+
let prefixed;
|
|
49
|
+
let anyReal;
|
|
50
|
+
for (const [stamp, n] of Object.entries(modelCounts)) {
|
|
51
|
+
if (NON_SPACE_STAMPS.has(stamp) || typeof n !== "number" || n <= 0)
|
|
52
|
+
continue;
|
|
53
|
+
anyReal ??= stamp;
|
|
54
|
+
if (stamp.includes("+searchprefix")) {
|
|
55
|
+
if (!prefixed || stamp.includes(":"))
|
|
56
|
+
prefixed = stamp;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
if (prefixed)
|
|
60
|
+
return prefixed;
|
|
61
|
+
if (anyReal) {
|
|
62
|
+
const bare = anyReal.includes(":") ? anyReal.slice(anyReal.indexOf(":") + 1) : anyReal;
|
|
63
|
+
return `${DEFAULT_ENGINE}:${bare}+searchprefix`;
|
|
64
|
+
}
|
|
65
|
+
return `${DEFAULT_ENGINE}:nomic-embed-text-v1.5-Q4_K_M+searchprefix`;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Split the corpus into current-space vs foreign-space counts. A bare
|
|
69
|
+
* stamp and its `gguf:` equivalent are the SAME space; `+searchprefix`
|
|
70
|
+
* vs bare-without-suffix are NOT (that split is this migration's payload).
|
|
71
|
+
*/
|
|
72
|
+
export function countStampSpaces(modelCounts, currentModelId) {
|
|
73
|
+
const currentSpace = normalizeStampForOutstanding(currentModelId);
|
|
74
|
+
let currentCount = 0;
|
|
75
|
+
let staleCount = 0;
|
|
76
|
+
const staleStamps = [];
|
|
77
|
+
for (const [raw, n] of Object.entries(modelCounts)) {
|
|
78
|
+
if (typeof n !== "number" || n <= 0)
|
|
79
|
+
continue;
|
|
80
|
+
const space = normalizeStampForOutstanding(raw);
|
|
81
|
+
if (space === null)
|
|
82
|
+
continue;
|
|
83
|
+
if (currentSpace !== null && space === currentSpace) {
|
|
84
|
+
currentCount += n;
|
|
85
|
+
}
|
|
86
|
+
else {
|
|
87
|
+
staleCount += n;
|
|
88
|
+
staleStamps.push(`${raw}:${n}`);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return { currentCount, staleCount, staleStamps, currentSpace };
|
|
92
|
+
}
|
|
93
|
+
function runnerAnnotation(input) {
|
|
94
|
+
const mig = input.migration;
|
|
95
|
+
const phase = input.cyclePhase;
|
|
96
|
+
if (input.lastCycleError) {
|
|
97
|
+
return ` last boot cycle failed (${input.lastCycleError})`;
|
|
98
|
+
}
|
|
99
|
+
if (phase === "idle") {
|
|
100
|
+
return " migration boot cycle never fired on this instance — no migration will run until this is resolved";
|
|
101
|
+
}
|
|
102
|
+
if (mig?.state === "running" || mig?.state === "checking" || mig?.state === "preflight" || mig?.state === "snapshotting" || mig?.state === "completing") {
|
|
103
|
+
const rem = typeof mig.rowsRemaining === "number" ? `, ${mig.rowsRemaining} remaining` : "";
|
|
104
|
+
return ` in progress (${mig.state}${rem})`;
|
|
105
|
+
}
|
|
106
|
+
if (mig?.state === "halted" || mig?.state === "failed") {
|
|
107
|
+
return ` ${mig.state}${mig.reason ? `: ${mig.reason}` : ""} — see \`flair doctor\``;
|
|
108
|
+
}
|
|
109
|
+
if (mig?.state === "completed" || phase === "done") {
|
|
110
|
+
return " the last cycle marked it complete without converging; it will retry automatically";
|
|
111
|
+
}
|
|
112
|
+
if (input.audience === "client") {
|
|
113
|
+
return " the Harper process applies it on its next migration cycle — this CLI does not";
|
|
114
|
+
}
|
|
115
|
+
return " it will apply automatically on this process's next migration cycle";
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Corpus-derived outstanding check. Returns `outstanding: false` when every
|
|
119
|
+
* real-space stamp matches the current space (or the store is empty of
|
|
120
|
+
* real embeddings). Runner bookkeeping cannot override a split corpus.
|
|
121
|
+
*/
|
|
122
|
+
export function describeStampOutstanding(input) {
|
|
123
|
+
const { currentCount, staleCount, staleStamps } = countStampSpaces(input.modelCounts, input.currentModelId);
|
|
124
|
+
if (staleCount === 0) {
|
|
125
|
+
return { outstanding: false, staleCount: 0, currentCount };
|
|
126
|
+
}
|
|
127
|
+
const list = staleStamps.join(", ");
|
|
128
|
+
const warning = `migration '${EMBEDDING_STAMP_ID}' is outstanding ` +
|
|
129
|
+
`(${staleCount} row${staleCount === 1 ? "" : "s"} still on ${list}` +
|
|
130
|
+
`${currentCount > 0 ? `, ${currentCount} already current` : ""}) — ` +
|
|
131
|
+
`cross-model search is unreliable and duplicate detection is inactive ` +
|
|
132
|
+
`until the re-embed completes;` +
|
|
133
|
+
runnerAnnotation(input);
|
|
134
|
+
return {
|
|
135
|
+
outstanding: true,
|
|
136
|
+
migrationId: EMBEDDING_STAMP_ID,
|
|
137
|
+
staleCount,
|
|
138
|
+
currentCount,
|
|
139
|
+
staleStamps,
|
|
140
|
+
warning,
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* Convergence predicate for `upgrade --target` / `deploy` post-verify
|
|
145
|
+
* (flair#1073). A split corpus is not converged. A halted/failed
|
|
146
|
+
* embedding-stamp is not converged. A still-running cycle is not
|
|
147
|
+
* converged. Missing migration state on a split corpus is not converged.
|
|
148
|
+
* An empty / already-current corpus is converged.
|
|
149
|
+
*/
|
|
150
|
+
export function stampMigrationConverged(input) {
|
|
151
|
+
const described = describeStampOutstanding(input);
|
|
152
|
+
if (described.outstanding) {
|
|
153
|
+
return { converged: false, detail: described.warning };
|
|
154
|
+
}
|
|
155
|
+
const mig = input.migration;
|
|
156
|
+
if (mig?.state === "halted" || mig?.state === "failed") {
|
|
157
|
+
return {
|
|
158
|
+
converged: false,
|
|
159
|
+
detail: `migration '${EMBEDDING_STAMP_ID}' ${mig.state}${mig.reason ? `: ${mig.reason}` : ""}`,
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
if (mig &&
|
|
163
|
+
(mig.state === "running" ||
|
|
164
|
+
mig.state === "checking" ||
|
|
165
|
+
mig.state === "preflight" ||
|
|
166
|
+
mig.state === "snapshotting" ||
|
|
167
|
+
mig.state === "completing")) {
|
|
168
|
+
return { converged: false, detail: `migration '${EMBEDDING_STAMP_ID}' still ${mig.state}` };
|
|
169
|
+
}
|
|
170
|
+
return { converged: true, detail: "embedding-stamp converged (corpus is a single current space)" };
|
|
171
|
+
}
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
* WHY THIS EXISTS: `resources/Memory.ts`'s `defaultVisibilityForDurability`
|
|
13
13
|
* (flair#509) only stamps `visibility` on a WRITE — a row that has sat
|
|
14
14
|
* untouched since before that slice shipped has no `visibility` field at
|
|
15
|
-
* all. `src/
|
|
15
|
+
* all. `src/commands/federation.ts`'s federation push filter (`isFederationPrivateVisibility`)
|
|
16
16
|
* excludes only `visibility === "private"` — by design, per
|
|
17
17
|
* `resources/memory-visibility.ts`'s migration invariant, an ABSENT
|
|
18
18
|
* `visibility` field is NOT private and must keep syncing exactly as
|
|
@@ -38,7 +38,7 @@
|
|
|
38
38
|
* EXTENDS that only resolves once a live Harper has already registered the
|
|
39
39
|
* base `Memory` resource, which unit tests (no real Harper; see
|
|
40
40
|
* `test/unit/migrations-visibility-backfill.test.ts`) cannot satisfy. Same
|
|
41
|
-
* reasoning, same shape, as `src/
|
|
41
|
+
* reasoning, same shape, as `src/commands/federation.ts`'s `isFederationPrivateVisibility`
|
|
42
42
|
* duplicating `resources/memory-visibility.ts`'s `isPrivateVisibility`
|
|
43
43
|
* rather than importing it (see that file's comment) — both are one-line,
|
|
44
44
|
* pure, and load-bearing enough that a comment pointing at the canonical
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* org-event-catchup-lib.ts — Harper-free OrgEvent catch-up filters + paging.
|
|
3
|
+
*
|
|
4
|
+
* Targeting / expiry / position-order live here so OrgEventCatchup and
|
|
5
|
+
* MemoryBootstrap share one definition (and unit tests can pin it without
|
|
6
|
+
* spinning Harper). Watermark storage is resources/agent-read-position.ts.
|
|
7
|
+
*/
|
|
8
|
+
import { comparePosition, createdAtFloorFromPosition, laterTimestamp, pageAfter, recordPosition, } from "./agent-read-position-lib.js";
|
|
9
|
+
export function eventTargetsParticipant(event, participantId) {
|
|
10
|
+
const targets = event.targetIds;
|
|
11
|
+
return !targets || targets.length === 0 || targets.includes(participantId);
|
|
12
|
+
}
|
|
13
|
+
export function eventIsExpired(event, now = new Date()) {
|
|
14
|
+
return Boolean(event.expiresAt && new Date(event.expiresAt) < now);
|
|
15
|
+
}
|
|
16
|
+
export function withEventPosition(event) {
|
|
17
|
+
return { ...event, position: recordPosition(event) };
|
|
18
|
+
}
|
|
19
|
+
export function catchupSeekTimestamp(after, since) {
|
|
20
|
+
return laterTimestamp(createdAtFloorFromPosition(after), since ?? null);
|
|
21
|
+
}
|
|
22
|
+
export function isCatchupEligible(event, opts) {
|
|
23
|
+
if (!event.createdAt)
|
|
24
|
+
return false;
|
|
25
|
+
if (opts.since && event.createdAt < opts.since)
|
|
26
|
+
return false;
|
|
27
|
+
if (comparePosition(recordPosition(event), opts.after) <= 0)
|
|
28
|
+
return false;
|
|
29
|
+
if (!eventTargetsParticipant(event, opts.participantId))
|
|
30
|
+
return false;
|
|
31
|
+
if (eventIsExpired(event, opts.now ?? new Date()))
|
|
32
|
+
return false;
|
|
33
|
+
return true;
|
|
34
|
+
}
|
|
35
|
+
export async function collectCatchupEvents(rows, opts) {
|
|
36
|
+
const results = [];
|
|
37
|
+
for await (const event of rows) {
|
|
38
|
+
if (!isCatchupEligible(event, opts))
|
|
39
|
+
continue;
|
|
40
|
+
results.push(withEventPosition(event));
|
|
41
|
+
}
|
|
42
|
+
results.sort((a, b) => comparePosition(a.position, b.position));
|
|
43
|
+
return results;
|
|
44
|
+
}
|
|
45
|
+
export function pageCatchupEvents(ordered, after, pageSize) {
|
|
46
|
+
return pageAfter(ordered, after, pageSize);
|
|
47
|
+
}
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* stamp-migration-verify.ts — post-deploy / `upgrade --target` check that
|
|
3
|
+
* embedding-stamp actually converged (flair#1073).
|
|
4
|
+
*
|
|
5
|
+
* Route verify (`verifyDeployServing`) proves the component is serving.
|
|
6
|
+
* That is not enough: a Fabric upgrade can serve new writes (prefixed
|
|
7
|
+
* stamps) while pre-flip rows stay stale, and `flair status` used to
|
|
8
|
+
* report only a generic mixed-models warning. This polls authenticated
|
|
9
|
+
* /HealthDetail until the corpus is a single current space (and
|
|
10
|
+
* embedding-stamp is not halted/failed), the same way route verify polls
|
|
11
|
+
* until /Memory is non-404.
|
|
12
|
+
*/
|
|
13
|
+
import { EMBEDDING_STAMP_ID, stampMigrationConverged, } from "./stamp-outstanding.js";
|
|
14
|
+
export const DEFAULT_STAMP_VERIFY_TIMEOUT_MS = 600_000;
|
|
15
|
+
export const STAMP_VERIFY_POLL_INTERVAL_MS = 2_000;
|
|
16
|
+
function basicAuthHeader(user, password) {
|
|
17
|
+
return "Basic " + Buffer.from(`${user}:${password}`, "utf8").toString("base64");
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Parse /HealthDetail into a verify snapshot. `memories: null` (HealthDetail
|
|
21
|
+
* 200 after the Memory walk threw) and a missing `modelCounts` field stay
|
|
22
|
+
* `undefined` — that is unread, not an empty corpus. An explicit
|
|
23
|
+
* `modelCounts: {}` is the valid empty-store case (Bugbot Medium on #1606).
|
|
24
|
+
*/
|
|
25
|
+
export function snapshotFromHealthDetail(body) {
|
|
26
|
+
const rec = body && typeof body === "object" ? body : {};
|
|
27
|
+
const memories = rec.memories && typeof rec.memories === "object" ? rec.memories : null;
|
|
28
|
+
const migrationsBlock = rec.migrations && typeof rec.migrations === "object" ? rec.migrations : {};
|
|
29
|
+
const modelCounts = memories && memories.modelCounts && typeof memories.modelCounts === "object"
|
|
30
|
+
? memories.modelCounts
|
|
31
|
+
: undefined;
|
|
32
|
+
const rawList = Array.isArray(migrationsBlock.migrations) ? migrationsBlock.migrations : undefined;
|
|
33
|
+
const migrations = rawList
|
|
34
|
+
?.filter((m) => !!m && typeof m === "object" && typeof m.id === "string")
|
|
35
|
+
.map((m) => ({
|
|
36
|
+
id: m.id,
|
|
37
|
+
state: String(m.state ?? ""),
|
|
38
|
+
rowsDone: typeof m.rowsDone === "number" ? m.rowsDone : undefined,
|
|
39
|
+
rowsRemaining: typeof m.rowsRemaining === "number" ? m.rowsRemaining : undefined,
|
|
40
|
+
reason: typeof m.reason === "string" ? m.reason : undefined,
|
|
41
|
+
}));
|
|
42
|
+
const warnings = Array.isArray(rec.warnings)
|
|
43
|
+
? rec.warnings
|
|
44
|
+
.map((w) => (w && typeof w === "object" && typeof w.message === "string" ? w.message : null))
|
|
45
|
+
.filter((m) => m !== null)
|
|
46
|
+
: undefined;
|
|
47
|
+
return {
|
|
48
|
+
modelCounts,
|
|
49
|
+
cyclePhase: typeof migrationsBlock.cyclePhase === "string" ? migrationsBlock.cyclePhase : undefined,
|
|
50
|
+
lastCycleError: typeof migrationsBlock.lastCycleError === "string" ? migrationsBlock.lastCycleError : migrationsBlock.lastCycleError === null ? null : undefined,
|
|
51
|
+
migrations,
|
|
52
|
+
warnings,
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Read the current-space id from the HealthDetail warning or from the
|
|
57
|
+
* dominant +searchprefix / gguf: stamp. Fallback: the most common real
|
|
58
|
+
* stamp — verify then treats any other space as stale.
|
|
59
|
+
*/
|
|
60
|
+
export function inferCurrentModelId(snapshot) {
|
|
61
|
+
const counts = snapshot.modelCounts ?? {};
|
|
62
|
+
let best = null;
|
|
63
|
+
let prefixed = null;
|
|
64
|
+
for (const [stamp, n] of Object.entries(counts)) {
|
|
65
|
+
if (stamp === "hash-512d" || typeof n !== "number" || n <= 0)
|
|
66
|
+
continue;
|
|
67
|
+
if (!best || n > best.n)
|
|
68
|
+
best = { stamp, n };
|
|
69
|
+
if (stamp.includes("+searchprefix") && (!prefixed || n > prefixed.n))
|
|
70
|
+
prefixed = { stamp, n };
|
|
71
|
+
}
|
|
72
|
+
return prefixed?.stamp ?? best?.stamp ?? "nomic-embed-text-v1.5-Q4_K_M+searchprefix";
|
|
73
|
+
}
|
|
74
|
+
export function evaluateStampSnapshot(snapshot) {
|
|
75
|
+
// Unread corpus (HealthDetail omitted modelCounts, including memories: null)
|
|
76
|
+
// is not an empty store. Reject before stampMigrationConverged can treat
|
|
77
|
+
// `undefined` as `{}` and report success (Bugbot Medium on #1606).
|
|
78
|
+
if (snapshot.modelCounts === undefined) {
|
|
79
|
+
return {
|
|
80
|
+
converged: false,
|
|
81
|
+
detail: "HealthDetail did not include memories.modelCounts",
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
const outstandingWarning = snapshot.warnings?.find((w) => w.includes(`migration '${EMBEDDING_STAMP_ID}' is outstanding`) || w.includes("duplicate detection is inactive"));
|
|
85
|
+
if (outstandingWarning) {
|
|
86
|
+
return { converged: false, detail: outstandingWarning };
|
|
87
|
+
}
|
|
88
|
+
const counts = snapshot.modelCounts;
|
|
89
|
+
const realStamps = Object.entries(counts).filter(([k, n]) => k !== "hash-512d" && typeof n === "number" && n > 0);
|
|
90
|
+
// Production getModelId() stamps +searchprefix. A corpus that is entirely
|
|
91
|
+
// on the pre-flip bare id is the #1073 failure mode — do not treat a
|
|
92
|
+
// single-space bare corpus as converged just because inferCurrentModelId
|
|
93
|
+
// would pick that majority stamp as "current."
|
|
94
|
+
if (realStamps.length > 0 && realStamps.every(([k]) => !k.includes("+searchprefix"))) {
|
|
95
|
+
const list = realStamps.map(([k, n]) => `${k}:${n}`).join(", ");
|
|
96
|
+
return {
|
|
97
|
+
converged: false,
|
|
98
|
+
detail: `migration '${EMBEDDING_STAMP_ID}' is outstanding — corpus has no +searchprefix stamp yet (${list})`,
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
const currentModelId = inferCurrentModelId(snapshot);
|
|
102
|
+
const migration = snapshot.migrations?.find((m) => m.id === EMBEDDING_STAMP_ID);
|
|
103
|
+
return stampMigrationConverged({
|
|
104
|
+
modelCounts: counts,
|
|
105
|
+
currentModelId,
|
|
106
|
+
migration,
|
|
107
|
+
cyclePhase: snapshot.cyclePhase,
|
|
108
|
+
lastCycleError: snapshot.lastCycleError,
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
async function readHealthDetail(baseUrl, auth, fetchImpl) {
|
|
112
|
+
const url = `${baseUrl.replace(/\/+$/, "")}/HealthDetail`;
|
|
113
|
+
try {
|
|
114
|
+
const res = await fetchImpl(url, {
|
|
115
|
+
headers: { Authorization: auth, Accept: "application/json" },
|
|
116
|
+
signal: AbortSignal.timeout(15_000),
|
|
117
|
+
});
|
|
118
|
+
if (!res.ok) {
|
|
119
|
+
return { ok: false, status: res.status, error: `GET /HealthDetail returned ${res.status}` };
|
|
120
|
+
}
|
|
121
|
+
const body = await res.json();
|
|
122
|
+
return { ok: true, status: res.status, snapshot: snapshotFromHealthDetail(body) };
|
|
123
|
+
}
|
|
124
|
+
catch (err) {
|
|
125
|
+
return { ok: false, status: 0, error: err?.message ?? String(err) };
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Poll /HealthDetail until embedding-stamp has converged, or throw.
|
|
130
|
+
* Skipped by the caller when verify is disabled (`--no-verify`).
|
|
131
|
+
*/
|
|
132
|
+
export async function verifyStampMigrationConverged(o) {
|
|
133
|
+
const { baseUrl, fabricUser, fabricPassword, timeoutMs = DEFAULT_STAMP_VERIFY_TIMEOUT_MS, pollIntervalMs = STAMP_VERIFY_POLL_INTERVAL_MS, fetchImpl = fetch, sleep = (ms) => new Promise((r) => setTimeout(r, ms)), onProgress, } = o;
|
|
134
|
+
const auth = basicAuthHeader(fabricUser, fabricPassword);
|
|
135
|
+
onProgress?.(`verifying ${EMBEDDING_STAMP_ID} converged on ${baseUrl}...`);
|
|
136
|
+
const deadline = Date.now() + timeoutMs;
|
|
137
|
+
let last = await readHealthDetail(baseUrl, auth, fetchImpl);
|
|
138
|
+
for (;;) {
|
|
139
|
+
if (last.ok && last.snapshot) {
|
|
140
|
+
const verdict = evaluateStampSnapshot(last.snapshot);
|
|
141
|
+
if (verdict.converged) {
|
|
142
|
+
onProgress?.(`embedding-stamp converged`);
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
onProgress?.(verdict.detail);
|
|
146
|
+
}
|
|
147
|
+
else {
|
|
148
|
+
onProgress?.(last.error ?? "HealthDetail not readable yet");
|
|
149
|
+
}
|
|
150
|
+
if (Date.now() >= deadline)
|
|
151
|
+
break;
|
|
152
|
+
await sleep(Math.min(pollIntervalMs, Math.max(0, deadline - Date.now())));
|
|
153
|
+
last = await readHealthDetail(baseUrl, auth, fetchImpl);
|
|
154
|
+
}
|
|
155
|
+
if (!last.ok) {
|
|
156
|
+
throw new Error(`deploy verification: embedding-stamp did not converge within ${timeoutMs}ms — ` +
|
|
157
|
+
`could not read /HealthDetail (${last.error ?? `HTTP ${last.status}`}). ` +
|
|
158
|
+
`The component may still be restarting; re-run without --no-verify, or ` +
|
|
159
|
+
`\`flair status --json\` and look for migration '${EMBEDDING_STAMP_ID}'.`);
|
|
160
|
+
}
|
|
161
|
+
const verdict = last.snapshot ? evaluateStampSnapshot(last.snapshot) : { detail: "no HealthDetail snapshot" };
|
|
162
|
+
throw new Error(`deploy verification: embedding-stamp did not converge within ${timeoutMs}ms — ${verdict.detail}`);
|
|
163
|
+
}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* stamp-outstanding.ts — CLI-side copy of the corpus-derived
|
|
3
|
+
* "is embedding-stamp still pending?" helper (flair#1073).
|
|
4
|
+
*
|
|
5
|
+
* INLINED, not imported from resources/: cross-boundary imports from src/
|
|
6
|
+
* into resources/ do not survive npm packaging. tsconfig.cli.json compiles
|
|
7
|
+
* with `rootDir: "src"`, so dist/cli.js has no resources/ module it can
|
|
8
|
+
* resolve at the same relative path. Same reason as entity-vocab-cli.ts
|
|
9
|
+
* and the getModelId() literals in src/cli.ts.
|
|
10
|
+
*
|
|
11
|
+
* Canonical module: resources/migrations/stamp-outstanding.ts (Harper
|
|
12
|
+
* HealthDetail / embedding-stamp). The two files MUST stay in sync —
|
|
13
|
+
* test/unit/stamp-outstanding-cli-parity.test.ts pins export names and a
|
|
14
|
+
* known-answer table. Drift fails CI rather than shipping.
|
|
15
|
+
*/
|
|
16
|
+
export const EMBEDDING_STAMP_ID = "embedding-stamp";
|
|
17
|
+
const NON_SPACE_STAMPS = new Set(["hash-512d"]);
|
|
18
|
+
const DEFAULT_ENGINE = "gguf";
|
|
19
|
+
export function normalizeStampForOutstanding(stamp) {
|
|
20
|
+
if (stamp == null)
|
|
21
|
+
return null;
|
|
22
|
+
const s = String(stamp).trim();
|
|
23
|
+
if (s === "" || NON_SPACE_STAMPS.has(s))
|
|
24
|
+
return null;
|
|
25
|
+
return s.includes(":") ? s : `${DEFAULT_ENGINE}:${s}`;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Current-space id for a client that only has `modelCounts` (no live
|
|
29
|
+
* `getModelId()`). Prefers an existing `+searchprefix` stamp so a healthy
|
|
30
|
+
* bare+`gguf:` pair of the SAME space does not invent a foreign current.
|
|
31
|
+
* A uniformly pre-flip corpus has no suffix — append `+searchprefix` so
|
|
32
|
+
* those rows read as stale.
|
|
33
|
+
*/
|
|
34
|
+
export function resolveCurrentModelId(modelCounts, explicit) {
|
|
35
|
+
const trimmed = explicit?.trim();
|
|
36
|
+
if (trimmed)
|
|
37
|
+
return trimmed;
|
|
38
|
+
let prefixed;
|
|
39
|
+
let anyReal;
|
|
40
|
+
for (const [stamp, n] of Object.entries(modelCounts)) {
|
|
41
|
+
if (NON_SPACE_STAMPS.has(stamp) || typeof n !== "number" || n <= 0)
|
|
42
|
+
continue;
|
|
43
|
+
anyReal ??= stamp;
|
|
44
|
+
if (stamp.includes("+searchprefix")) {
|
|
45
|
+
if (!prefixed || stamp.includes(":"))
|
|
46
|
+
prefixed = stamp;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
if (prefixed)
|
|
50
|
+
return prefixed;
|
|
51
|
+
if (anyReal) {
|
|
52
|
+
const bare = anyReal.includes(":") ? anyReal.slice(anyReal.indexOf(":") + 1) : anyReal;
|
|
53
|
+
return `${DEFAULT_ENGINE}:${bare}+searchprefix`;
|
|
54
|
+
}
|
|
55
|
+
return `${DEFAULT_ENGINE}:nomic-embed-text-v1.5-Q4_K_M+searchprefix`;
|
|
56
|
+
}
|
|
57
|
+
export function countStampSpaces(modelCounts, currentModelId) {
|
|
58
|
+
const currentSpace = normalizeStampForOutstanding(currentModelId);
|
|
59
|
+
let currentCount = 0;
|
|
60
|
+
let staleCount = 0;
|
|
61
|
+
const staleStamps = [];
|
|
62
|
+
for (const [raw, n] of Object.entries(modelCounts)) {
|
|
63
|
+
if (typeof n !== "number" || n <= 0)
|
|
64
|
+
continue;
|
|
65
|
+
const space = normalizeStampForOutstanding(raw);
|
|
66
|
+
if (space === null)
|
|
67
|
+
continue;
|
|
68
|
+
if (currentSpace !== null && space === currentSpace) {
|
|
69
|
+
currentCount += n;
|
|
70
|
+
}
|
|
71
|
+
else {
|
|
72
|
+
staleCount += n;
|
|
73
|
+
staleStamps.push(`${raw}:${n}`);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
return { currentCount, staleCount, staleStamps, currentSpace };
|
|
77
|
+
}
|
|
78
|
+
function runnerAnnotation(input) {
|
|
79
|
+
const mig = input.migration;
|
|
80
|
+
const phase = input.cyclePhase;
|
|
81
|
+
if (input.lastCycleError) {
|
|
82
|
+
return ` last boot cycle failed (${input.lastCycleError})`;
|
|
83
|
+
}
|
|
84
|
+
if (phase === "idle") {
|
|
85
|
+
return " migration boot cycle never fired on this instance — no migration will run until this is resolved";
|
|
86
|
+
}
|
|
87
|
+
if (mig?.state === "running" || mig?.state === "checking" || mig?.state === "preflight" || mig?.state === "snapshotting" || mig?.state === "completing") {
|
|
88
|
+
const rem = typeof mig.rowsRemaining === "number" ? `, ${mig.rowsRemaining} remaining` : "";
|
|
89
|
+
return ` in progress (${mig.state}${rem})`;
|
|
90
|
+
}
|
|
91
|
+
if (mig?.state === "halted" || mig?.state === "failed") {
|
|
92
|
+
return ` ${mig.state}${mig.reason ? `: ${mig.reason}` : ""} — see \`flair doctor\``;
|
|
93
|
+
}
|
|
94
|
+
if (mig?.state === "completed" || phase === "done") {
|
|
95
|
+
return " the last cycle marked it complete without converging; it will retry automatically";
|
|
96
|
+
}
|
|
97
|
+
if (input.audience === "client") {
|
|
98
|
+
return " the Harper process applies it on its next migration cycle — this CLI does not";
|
|
99
|
+
}
|
|
100
|
+
return " it will apply automatically on this process's next migration cycle";
|
|
101
|
+
}
|
|
102
|
+
export function describeStampOutstanding(input) {
|
|
103
|
+
const { currentCount, staleCount, staleStamps } = countStampSpaces(input.modelCounts, input.currentModelId);
|
|
104
|
+
if (staleCount === 0) {
|
|
105
|
+
return { outstanding: false, staleCount: 0, currentCount };
|
|
106
|
+
}
|
|
107
|
+
const list = staleStamps.join(", ");
|
|
108
|
+
const warning = `migration '${EMBEDDING_STAMP_ID}' is outstanding ` +
|
|
109
|
+
`(${staleCount} row${staleCount === 1 ? "" : "s"} still on ${list}` +
|
|
110
|
+
`${currentCount > 0 ? `, ${currentCount} already current` : ""}) — ` +
|
|
111
|
+
`cross-model search is unreliable and duplicate detection is inactive ` +
|
|
112
|
+
`until the re-embed completes;` +
|
|
113
|
+
runnerAnnotation(input);
|
|
114
|
+
return {
|
|
115
|
+
outstanding: true,
|
|
116
|
+
migrationId: EMBEDDING_STAMP_ID,
|
|
117
|
+
staleCount,
|
|
118
|
+
currentCount,
|
|
119
|
+
staleStamps,
|
|
120
|
+
warning,
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
export function stampMigrationConverged(input) {
|
|
124
|
+
const described = describeStampOutstanding(input);
|
|
125
|
+
if (described.outstanding) {
|
|
126
|
+
return { converged: false, detail: described.warning };
|
|
127
|
+
}
|
|
128
|
+
const mig = input.migration;
|
|
129
|
+
if (mig?.state === "halted" || mig?.state === "failed") {
|
|
130
|
+
return {
|
|
131
|
+
converged: false,
|
|
132
|
+
detail: `migration '${EMBEDDING_STAMP_ID}' ${mig.state}${mig.reason ? `: ${mig.reason}` : ""}`,
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
if (mig &&
|
|
136
|
+
(mig.state === "running" ||
|
|
137
|
+
mig.state === "checking" ||
|
|
138
|
+
mig.state === "preflight" ||
|
|
139
|
+
mig.state === "snapshotting" ||
|
|
140
|
+
mig.state === "completing")) {
|
|
141
|
+
return { converged: false, detail: `migration '${EMBEDDING_STAMP_ID}' still ${mig.state}` };
|
|
142
|
+
}
|
|
143
|
+
return { converged: true, detail: "embedding-stamp converged (corpus is a single current space)" };
|
|
144
|
+
}
|