@smartmemory/compose 0.4.1 → 0.5.0

package/lib/fluid/import-ideabox.js

@@ -21,6 +21,14 @@ import { readFileSync } from 'node:fs';
 
 import { parseIdeabox } from '../ideabox.js';
 import { toRecordTimestamp } from './ideabox-dates.js';
+import {
+  IdeaboxSourceChangedDuringImport,
+  closeManifest,
+  hashMarkdown,
+  openManifest,
+} from './ideabox-manifest.js';
+import { writePreamble } from './ideabox-preamble.js';
+import { assertIdeaboxReadable } from './ideabox-readable.js';
 import { KIND } from './provider.js';
 
 /** Markdown status token → canonical fluid status. */
@@ -59,6 +67,12 @@ function toPriority(raw) {
 export async function importIdeabox(provider, { markdown, path, dryRun = false } = {}) {
   const source = markdown ?? readFileSync(path, 'utf8');
   const parsed = parseIdeabox(source);
+  // FU-2. This function is directly callable, and it used to read a failed
+  // parse as an empty document — importing only the subset it recognised and
+  // leaving the rest to be projected away by the next render. Same shape as the
+  // bug that destroyed 18 ideas, one level up: the guard was on the destructive
+  // path, not on the parse.
+  assertIdeaboxReadable(parsed, path ?? '(in-memory ideabox)');
 
   const provenance = { origin: 'import:ideabox' };
 
@@ -71,6 +85,43 @@ export async function importIdeabox(provider, { markdown, path, dryRun = false }
   const skipped = [];
   const clusterHandles = [];
 
+  // FU-1: THE INTENTION, WRITTEN DOWN BEFORE THE FIRST WRITE.
+  //
+  // Records are created one at a time. A crash on idea 2 leaves ideas 3..N
+  // never issued, so they carry no event, so the gate reads them as hand-added
+  // strays and refuses — and every recovery it names runs the same gate. The
+  // log can only testify about handles the import REACHED; the unattempted tail
+  // leaves no trace, so it has to be declared in advance or it cannot be
+  // recovered at all.
+  //
+  // Written before the first `createRecord` of either population, and removed
+  // only on success, so "open" means "an import started here and did not
+  // finish". A dry run declares nothing because it writes nothing.
+  const plannedHandles = [
+    ...(parsed.ideas ?? []).map((i) => i.id),
+    ...(parsed.killed ?? []).map((i) => i.id),
+  ];
+  if (!dryRun && plannedHandles.length) {
+    openManifest(provider, path, {
+      markdown: source,
+      planned: plannedHandles,
+      plannedClusters: (parsed.clusters ?? []).map((c) => c.name),
+    });
+  }
+
+  // THE DOCUMENT AROUND THE IDEAS (FU-4).
+  //
+  // A project's own title and introduction are content too, and the projection
+  // used to replace them with the standard template on the first render after
+  // migration. Captured here, at the one moment the source document is
+  // authoritative, from the PARSER'S OWN output rather than a second scan of
+  // the same text. It goes to a TRACKED sibling of the ideabox,
+  // `<ideabox>.preamble.md` — not beside the migration manifest, which is
+  // gitignored, because a tracked projection generated from untracked input
+  // loses the heading on every clone that did not run the migration. See
+  // `ideabox-preamble.js`.
+  if (!dryRun) writePreamble(path, parsed.preamble);
+
   // ---- clusters first: members reference them by handle --------------------
   const clusterHandleByName = new Map();
   for (const cluster of parsed.clusters ?? []) {
@@ -112,52 +163,12 @@ export async function importIdeabox(provider, { markdown, path, dryRun = false }
       skipped.push(idea.id);
       continue;
     }
-    const clusterHandle = idea.cluster ? clusterHandleByName.get(idea.cluster) ?? null : null;
-    const clusterOrder = idea.cluster
-      ? (parsed.clusters ?? []).find((c) => c.name === idea.cluster)?.order ?? null
-      : null;
-
-    const record = {
-      kind: KIND.IDEA,
-      // Verbatim. The whole point of the caller-supplied handle path.
-      handle: idea.id,
-      title: idea.title,
-      body: idea.description ?? '',
-      status: killed ? 'killed' : toStatus(idea.status),
-      // Keep the author's token when the canonical enum cannot hold it, so a
-      // closed enum does not quietly flatten `RE-AIMED (2026-07-21)` to `NEW`.
-      status_label: STATUS_MAP[String(idea.status ?? '').trim().toUpperCase()]
-        ? null
-        : (idea.status || null),
-      priority: toPriority(idea.priority),
-      // Carried, not dropped. `parseIdeabox` already validates both against
-      // their enums and yields null otherwise (`lib/ideabox.js:304-311`), so
-      // there is nothing to re-check here — but omitting them is not a harmless
-      // gap. This function is the first-use migration gate every upgrading
-      // install runs, and the render that follows it rewrites the markdown from
-      // the records. A dropped field is therefore deleted from the user's file
-      // on upgrade, silently, with no way back. No idea in THIS repo carries
-      // either, which is exactly why it went unnoticed.
-      effort: idea.effort ?? null,
-      impact: idea.impact ?? null,
-      cluster: clusterHandle,
-      cluster_order: clusterOrder,
-      tags: idea.tags ?? [],
-      source: idea.source || null,
-      links: idea.mapsTo ? [{ type: 'maps_to', target: idea.mapsTo }] : [],
-      killed: (killed || idea.killedReason)
-        ? {
-          at: toRecordTimestamp(idea.killedDate),
-          reason: idea.killedReason || 'reason not recorded in markdown',
-        }
-        : null,
-      discussion: (idea.discussion ?? []).map((d) => ({
-        at: toRecordTimestamp(d.date),
-        text: d.text ?? '',
-        author: d.author ?? null,
-      })),
+    const record = ideaToRecord(idea, {
+      killed,
+      parsed,
+      clusterHandleByName,
       provenance,
-    };
+    });
 
     if (dryRun) {
       imported.push(idea.id);
@@ -175,6 +186,36 @@ export async function importIdeabox(provider, { markdown, path, dryRun = false }
     imported.push(idea.id);
   }
 
+  // THE DOCUMENT IS STILL THE USER'S UNTIL THIS FINISHES.
+  //
+  // Every check up to here compares IDs, and an edit to an idea that ALREADY
+  // has a record changes no ID at all: the assessment sees a file whose handles
+  // are all known, calls it consistent, and the projection replaces the edited
+  // body with the one imported from the version read at the start. The edit is
+  // destroyed and it existed nowhere else.
+  //
+  // The scope is what makes this different from a hand edit to generated
+  // output. DURING the migration this file is the source document — nothing in
+  // it has reached the store yet — so an edit to it is unrecoverable content.
+  // AFTER the migration it is output, and discarding hand edits is exactly what
+  // `render` is for (`lib/ideabox-cli.js`); that contract is untouched. The
+  // open manifest is precisely the marker that separates the two states.
+  //
+  // Failing here leaves the manifest OPEN on purpose. Nothing has been
+  // destroyed, and the next run meets the refusal that already exists for this
+  // shape: the hash no longer matches, so the gate stops with
+  // `IDEABOX_MANIFEST_STALE` naming the mid-migration edit.
+  if (!dryRun && path && plannedHandles.length) {
+    if (hashMarkdown(readFileSync(path, 'utf8')) !== hashMarkdown(source)) {
+      throw new IdeaboxSourceChangedDuringImport(path);
+    }
+  }
+
+  // The import completed. Closing the manifest is what makes an OPEN one mean
+  // "interrupted" — and it is what keeps the protection intact, since a
+  // hand-added idea after a completed migration must still be refused.
+  if (!dryRun) closeManifest(provider, path);
+
   return {
     imported,
     skipped,
@@ -184,3 +225,105 @@ export async function importIdeabox(provider, { markdown, path, dryRun = false }
     alreadyImported: imported.length === 0 && skipped.length > 0,
   };
 }
+
+/**
+ * One markdown idea → one fluid record. THE ONLY MAPPING.
+ *
+ * Extracted from the import loop so the recovery path (`ideabox-recover.js`)
+ * can reconcile an already-imported record against the current file using the
+ * SAME rules the import used. A second mapping of this shape is the two-readers
+ * pattern that produced this entire bug class: the two would agree on the day
+ * they were written and drift apart on the first field either one gained.
+ *
+ * Pure. No provider, no I/O — the caller resolves cluster handles and decides
+ * whether to create or patch.
+ *
+ * @param {object} idea a `parseIdeabox` idea
+ * @param {object} ctx
+ * @param {boolean} ctx.killed whether it came from the killed section
+ * @param {object} ctx.parsed the whole parse, for cluster order
+ * @param {Map<string,string>} ctx.clusterHandleByName resolved cluster handles
+ * @param {object} ctx.provenance stamped on every record this writes
+ */
+export function ideaToRecord(idea, { killed = false, parsed, clusterHandleByName, provenance }) {
+  const clusterHandle = idea.cluster ? clusterHandleByName.get(idea.cluster) ?? null : null;
+  const clusterOrder = idea.cluster
+    ? (parsed.clusters ?? []).find((c) => c.name === idea.cluster)?.order ?? null
+    : null;
+
+  return {
+    kind: KIND.IDEA,
+    // Verbatim. The whole point of the caller-supplied handle path.
+    handle: idea.id,
+    title: idea.title,
+    body: idea.description ?? '',
+    status: killed ? 'killed' : toStatus(idea.status),
+    // Keep the author's token when the canonical enum cannot hold it, so a
+    // closed enum does not quietly flatten `RE-AIMED (2026-07-21)` to `NEW`.
+    status_label: STATUS_MAP[String(idea.status ?? '').trim().toUpperCase()]
+      ? null
+      : (idea.status || null),
+    priority: toPriority(idea.priority),
+    // Carried, not dropped. `parseIdeabox` already validates both against
+    // their enums and yields null otherwise (`lib/ideabox.js:304-311`), so
+    // there is nothing to re-check here — but omitting them is not a harmless
+    // gap. This function is the first-use migration gate every upgrading
+    // install runs, and the render that follows it rewrites the markdown from
+    // the records. A dropped field is therefore deleted from the user's file
+    // on upgrade, silently, with no way back. No idea in THIS repo carries
+    // either, which is exactly why it went unnoticed.
+    effort: idea.effort ?? null,
+    impact: idea.impact ?? null,
+    cluster: clusterHandle,
+    cluster_order: clusterOrder,
+    tags: idea.tags ?? [],
+    source: idea.source || null,
+    // Whatever the parser could not name, kept rather than discarded — MINUS
+    // the fields that have a typed home. `**Promoted to:**` is one the legacy
+    // parser does not know, so it lands in `_extraLines`; carrying it as an
+    // opaque extra puts it beyond the reach of `promoteIdea`, which filters
+    // stale `promoted_to` links before adding the new one
+    // (`ideabox-ops.js`). The record would then render BOTH the old target and
+    // the current one. A field with a typed representation must be imported
+    // into it, not carried around it.
+    extra_fields: extrasWithoutTypedFields(idea._extraLines ?? []),
+    links: [
+      ...(promotedTargetOf(idea._extraLines ?? [])
+        ? [{ type: 'promoted_to', target: promotedTargetOf(idea._extraLines ?? []) }]
+        : []),
+      ...(idea.mapsTo ? [{ type: 'maps_to', target: idea.mapsTo }] : []),
+    ],
+    killed: (killed || idea.killedReason)
+      ? {
+        at: toRecordTimestamp(idea.killedDate),
+        reason: idea.killedReason || 'reason not recorded in markdown',
+      }
+      : null,
+    discussion: (idea.discussion ?? []).map((d) => ({
+      at: toRecordTimestamp(d.date),
+      text: d.text ?? '',
+      author: d.author ?? null,
+    })),
+    provenance,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Typed fields hiding in the unrecognised pile
+// ---------------------------------------------------------------------------
+
+/** `**Promoted to:** X` written by hand, which the parser does not recognise. */
+const PROMOTED_TO_RE = /^\*\*Promoted to:\*\*\s*(.+)$/;
+
+function promotedTargetOf(extraLines) {
+  for (const line of extraLines) {
+    const m = String(line).match(PROMOTED_TO_RE);
+    if (m) return m[1].trim();
+  }
+  return null;
+}
+
+/** Drop the lines lifted into typed fields so they are not rendered twice. */
+function extrasWithoutTypedFields(extraLines) {
+  return extraLines.filter((l) => !PROMOTED_TO_RE.test(String(l)));
+}

package/lib/fluid/local-provider.js

@@ -397,6 +397,12 @@ export class LocalFluidProvider extends FluidProvider {
       cluster_order: input.cluster_order ?? null,
       tags: input.tags ?? [],
       source: input.source ?? null,
+      // Unrecognised hand-authored markdown fields, carried verbatim. This
+      // literal is one of FOUR places the record's fields are enumerated (the
+      // contract, `record-shape.js`, and both providers); a field missing from
+      // any one of them is silently dropped on write
+      // (COMP-IDEABOX-MIGRATE-DIALECT).
+      extra_fields: input.extra_fields ?? [],
       links: input.links ?? [],
       killed: input.killed ?? null,
       discussion: input.discussion ?? [],

package/lib/fluid/portfolio.js

@@ -0,0 +1,255 @@
+/**
+ * lib/fluid/portfolio.js — the cross-product aggregator (COMP-FOH FOH-7).
+ *
+ * WHAT THIS IS, AND WHAT IT DELIBERATELY IS NOT
+ * ---------------------------------------------
+ * A consumer-side aggregator that sits ABOVE the provider seam and fans a single
+ * question out across N independently-configured products.
+ *
+ * It is **not** a `FluidProvider` and must not be registered as one. A provider
+ * implies a store, and this has none; and putting fan-out below the seam would
+ * place it under per-workspace capability semantics, where "does this support
+ * recall" stops having a single answer. Members may use different providers —
+ * that is the point of aggregating above the seam rather than below it.
+ *
+ * PARTIALITY IS THE NORMAL CASE
+ * -----------------------------
+ * With N independently-configured products reached over N connections, some
+ * subset being unavailable is the expected steady state, not an error path. The
+ * contract is therefore that **every absent source is NAMED**: a portfolio answer
+ * that quietly covers three products when the user declared five is worse than
+ * no answer, because nothing in it looks wrong.
+ *
+ * The one thing that is an error is EVERY member failing. An empty result set
+ * and a total outage must never be presented identically.
+ */
+
+import { parsePortfolioConfig } from './factory.js';
+import { getFluidWorkspaceId } from '../maya-config.js';
+import { fluidProviderFor } from './factory.js';
+import { KIND, SEMANTIC_CAP } from './provider.js';
+
+/**
+ * Per-member deadline.
+ *
+ * Explicit, and not inherited: the colleague composer's 35s `withDeadline` wraps
+ * only its three findings calls, so there is no ambient whole-turn bound to sit
+ * inside (blueprint C2). A portfolio turn costs one round trip per member with no
+ * server-side batching, so an unbounded member holds the whole turn open.
+ */
+export const MEMBER_DEADLINE_MS = 20_000;
+
+/** How many records a storage-only member contributes when it cannot search. */
+const LISTED_FALLBACK_LIMIT = 25;
+
+/**
+ * @typedef {object} PortfolioSource
+ * @property {string} id    the declared label
+ * @property {string} root  the member's absolute root — carried because two
+ *   products can hold the same handle, and the id alone is a name the user chose
+ *   while the root is what actually disambiguates
+ * @property {import('./provider.js').RecallHit[]} hits
+ * @property {boolean} [listedNotSearched] true when the member could not search
+ *   and these are its records listed, never a synthesized query result
+ */
+
+/**
+ * Construct one provider per declared member, concurrently.
+ *
+ * A member that cannot be opened is an omission, never a thrown turn: the whole
+ * point is that one broken product does not silence the others.
+ *
+ * @param {string} cwd the declaring root
+ * @returns {Promise<{sources: Array<{id, root, provider}>, omissions: string[]}>}
+ */
+export async function openPortfolio(cwd) {
+  const config = parsePortfolioConfig(cwd);
+  if (!config) return { sources: [], omissions: [], declared: 0 };
+
+  const settled = await Promise.allSettled(
+    config.members.map(async (m) => ({ ...m, provider: await fluidProviderFor(m.root) })),
+  );
+
+  const sources = [];
+  const omissions = [];
+  settled.forEach((outcome, i) => {
+    const member = config.members[i];
+    if (outcome.status === 'fulfilled') sources.push(outcome.value);
+    else omissions.push(`${member.id} misconfigured: ${reasonOf(outcome.reason)}`);
+  });
+  return { sources, omissions, declared: config.members.length };
+}
+
+/**
+ * Ask every member the same question.
+ *
+ * @param {{sources: Array<{id, root, provider}>, omissions: string[]}} portfolio
+ * @param {string} query
+ * @param {{deadlineMs?: number, limit?: number}} [opts]
+ * @returns {Promise<{sources: PortfolioSource[], omissions: string[]}>}
+ */
+export async function recallAcrossPortfolio(portfolio, query, opts = {}) {
+  const deadlineMs = opts.deadlineMs ?? MEMBER_DEADLINE_MS;
+  const omissions = [...portfolio.omissions];
+
+  const settled = await Promise.allSettled(
+    portfolio.sources.map((source) => withDeadline(askOne(source, query, opts), deadlineMs, source.id)),
+  );
+
+  const sources = [];
+  settled.forEach((outcome, i) => {
+    const source = portfolio.sources[i];
+    if (outcome.status === 'rejected') {
+      omissions.push(`${source.id} ${classify(outcome.reason)}`);
+      return;
+    }
+    const { hits, listedNotSearched, omission } = outcome.value;
+    if (omission) omissions.push(omission);
+    sources.push({
+      id: source.id,
+      root: source.root,
+      hits,
+      ...(listedNotSearched ? { listedNotSearched: true } : {}),
+    });
+  });
+
+  // An empty answer and a total outage must not look the same. This is the one
+  // partiality that is NOT the normal case.
+  const declared = portfolio.declared ?? portfolio.sources.length;
+  if (!sources.length && declared > 0) {
+    const err = new Error(
+      `compose: every portfolio member failed — ${omissions.join('; ')}`,
+    );
+    err.code = 'PORTFOLIO_ALL_FAILED';
+    err.omissions = omissions;
+    throw err;
+  }
+
+  return { sources, omissions };
+}
+
+// ---------------------------------------------------------------------------
+// Internals
+// ---------------------------------------------------------------------------
+
+/**
+ * One member's contribution.
+ *
+ * A member that cannot search is not skipped and is not faked: it contributes
+ * what it genuinely has — its records, listed — flagged as such, plus a named
+ * omission. Returning its list as if it were a query result would be the silent
+ * substitution this feature exists to avoid.
+ */
+async function askOne(source, query, opts) {
+  const { provider, id } = source;
+  // The DECLARED constant, never a bare string: `has()` is a plain Set lookup
+  // on `capabilities()`, and the FOH-7 live-fire (2026-09-06) found `'recall'`
+  // here against a provider declaring `'RECALL'` — every member, SmartMemory
+  // included, silently took the listed-not-searched path and the suite stayed
+  // green because its stubs replaced `has()` outright.
+  const canRecall = typeof provider.has === 'function' ? provider.has(SEMANTIC_CAP.RECALL) : true;
+
+  if (!canRecall) {
+    const records = await provider.listRecords({ kind: KIND.IDEA });
+    // BY RECENCY, before the limit. `listRecords` returns the local provider's
+    // storage order (cluster, then handle number), so slicing it directly hands
+    // back a product's OLDEST ideas — an established product would omit its
+    // newest decisions every time, which is the worst possible subset to show
+    // for a question about what was recently decided.
+    const recent = [...records].sort((a, b) =>
+      String(b.updated_at ?? '').localeCompare(String(a.updated_at ?? '')));
+    return {
+      hits: recent.slice(0, opts.limit ?? LISTED_FALLBACK_LIMIT).map((record) => ({
+        handle: record.handle,
+        // Null, never 0: this member did not rank anything, and 0 would read as
+        // "ranked, and terrible" (see RecallHit in provider.js).
+        score: null,
+        record,
+      })),
+      listedNotSearched: true,
+      omission: `${id} cannot search (no recall capability) — listed its records instead`,
+    };
+  }
+
+  const hits = await provider.recall(query, opts.limit ? { limit: opts.limit } : {});
+  return { hits: hits ?? [], listedNotSearched: false, omission: null };
+}
+
+/** Reject after `ms`, naming the member so the omission can name it too. */
+function withDeadline(promise, ms, id) {
+  let timer;
+  const bell = new Promise((_, reject) => {
+    timer = setTimeout(() => {
+      const err = new Error(`did not answer within ${ms}ms`);
+      err.code = 'PORTFOLIO_MEMBER_TIMEOUT';
+      err.memberId = id;
+      reject(err);
+    }, ms);
+  });
+  return Promise.race([promise, bell]).finally(() => clearTimeout(timer));
+}
+
+const reasonOf = (err) => String(err?.message ?? err ?? 'unknown').slice(0, 160);
+
+/**
+ * Turn a failure into the omission's reason class.
+ *
+ * The vocabulary reuses wording the cockpit already uses ("SmartMemory
+ * unreachable", `RecallTab.jsx:29`) rather than coining a parallel one, and the
+ * 403 branch depends on `scopeError` being retained on the error — see the
+ * blueprint's D1.
+ */
+function classify(err) {
+  if (err?.code === 'PORTFOLIO_MEMBER_TIMEOUT') return `unreachable: ${reasonOf(err)}`;
+  const status = typeof err?.status === 'number' ? err.status : null;
+  if (status === 403 || status === 401) {
+    const scope = err?.scopeError;
+    if (scope === 'not-a-member') return 'unauthorized: not a member of that workspace';
+    if (scope === 'missing-scope') return 'unauthorized: the key lacks the required scope';
+    return 'unauthorized: reason undetermined';
+  }
+  if (status === 0 || /ECONNREFUSED|ENOTFOUND|fetch failed|network/i.test(reasonOf(err))) {
+    return `unreachable: ${reasonOf(err)}`;
+  }
+  return `unavailable: ${reasonOf(err)}`;
+}
+
+/**
+ * Refuse a portfolio where any member's fluid workspace is the colleague's own.
+ *
+ * The existing guard (`lib/maya-identity.js`) compares Maya's identity claim
+ * against ONE configured workspace — the declaring root's. A portfolio adds N
+ * more workspaces the turn will read from, and none of them was checked: the
+ * shallow-binding isolation this protects is not a property of the declaring
+ * root, it is a property of every workspace the turn touches. Without it a
+ * member configured at Maya's own workspace reaches a successful chat, which is
+ * the isolation failure the declaring-root check exists to prevent, arriving
+ * through a door that check does not cover.
+ *
+ * Deliberately NOT member-vs-member: two declared products may legitimately
+ * share a workspace, and refusing that would be our policy rather than a
+ * required invariant.
+ *
+ * @param {string} cwd the declaring root
+ * @param {string|null} identityWorkspaceId Maya's own verified workspace
+ * @throws {Error} code PORTFOLIO_WORKSPACE_COLLISION
+ */
+export function assertMemberWorkspacesDistinct(cwd, identityWorkspaceId) {
+  if (!identityWorkspaceId) return;
+  const config = parsePortfolioConfig(cwd);
+  if (!config) return;
+
+  const collisions = config.members
+    .filter((m) => getFluidWorkspaceId(m.root) === identityWorkspaceId)
+    .map((m) => m.id);
+
+  if (collisions.length) {
+    const err = new Error(
+      `compose: portfolio member(s) ${collisions.join(', ')} are configured at the colleague's own ` +
+      `workspace (${identityWorkspaceId}) — a turn cannot read the memory it is writing from`,
+    );
+    err.code = 'PORTFOLIO_WORKSPACE_COLLISION';
+    err.collisions = collisions;
+    throw err;
+  }
+}

package/lib/fluid/record-shape.js

@@ -204,6 +204,13 @@ export function normalizeRecord(record) {
     cluster_order: record.cluster_order ?? null,
     tags: Array.isArray(record.tags) ? [...record.tags] : [],
     source: record.source ?? null,
+    // Hand-authored field lines the markdown parser did not recognise. Carried
+    // verbatim so the upgrade path is LOSSLESS: this shape is an allowlist, so
+    // a field absent here is deleted from the user's ideabox by the first
+    // render after migration, silently and with no way back
+    // (COMP-IDEABOX-MIGRATE-DIALECT). `effort` and `impact` were rescued from
+    // exactly this fate one slice earlier; this is the general case they left.
+    extra_fields: Array.isArray(record.extra_fields) ? [...record.extra_fields] : [],
     links: Array.isArray(record.links) ? record.links.map((l) => ({ ...l })) : [],
     killed: record.killed ? { ...record.killed } : null,
     discussion: Array.isArray(record.discussion) ? record.discussion.map((d) => ({ ...d })) : [],