@smartmemory/compose 0.3.7 → 0.4.0

package/lib/fluid/factory.js

@@ -0,0 +1,167 @@
+/**
+ * lib/fluid/factory.js — configured fluid-store provider selection.
+ *
+ * Mirrors `lib/tracker/factory.js` in config handling (absent means default,
+ * malformed means fail loud) and DELIBERATELY DIVERGES from it in one respect:
+ * there is no `withFallback` proxy here.
+ *
+ * The tracker seam wraps its active provider so that an entity the provider
+ * cannot store falls through to the local one. That is right for the tracker,
+ * where the fallback is STORAGE and the substituted answer is equally true.
+ * It is wrong here. `PROVIDER-SEAM` forbids it: "a provider without a
+ * capability lacks it visibly; nothing fakes it." A fallback that answered
+ * `recall()` or `challenge()` from the floor would return a real-looking result
+ * produced by machinery that does not exist, which is worse than an error —
+ * the caller cannot tell the difference, and neither can the user.
+ *
+ * So capability absence propagates as `FluidCapabilityUnavailable` from
+ * `FluidProvider.require()`, and surfaces render it as a funnel ("challenge:
+ * connect SmartMemory") rather than an empty state.
+ */
+
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+import { getSmartmemoryConfig } from '../smartmemory-config.js';
+import { FluidConfigError, MUTATION_SCOPE, mutationScopeAtLeast } from './provider.js';
+import { LocalFluidProvider } from './local-provider.js';
+import { SmartMemoryFluidProvider } from './smartmemory-provider.js';
+
+/**
+ * Read `.compose/compose.json` → `fluid`.
+ * Absent file or absent key → local floor. That is a valid, supported
+ * configuration (the zero-install default), not a misconfiguration.
+ */
+function loadFluidConfig(cwd) {
+  const p = join(cwd, '.compose/compose.json');
+  if (!existsSync(p)) return { provider: 'local' };
+
+  let parsed;
+  try {
+    parsed = JSON.parse(readFileSync(p, 'utf8'));
+  } catch (e) {
+    // The file EXISTS but is malformed. Falling back silently here would mask a
+    // typo in the user's config and quietly downgrade them to the floor —
+    // meaning their configured semantic capabilities would vanish with no
+    // signal. Fail loud.
+    throw new FluidConfigError(
+      `compose: fluid config at ${p} contains invalid JSON — ${e.message}`,
+      { path: p }
+    );
+  }
+
+  // Valid JSON is not a valid config. `[]`, `"smartmemory"` and `42` all parse,
+  // and reading `.fluid` off them yields undefined — which would silently select
+  // the floor, exactly the quiet downgrade this function exists to prevent.
+  // `null` would throw a bare TypeError instead of a config error.
+  if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
+    throw new FluidConfigError(
+      `compose: fluid config at ${p} must be a JSON object ` +
+      `(got ${parsed === null ? 'null' : Array.isArray(parsed) ? 'array' : typeof parsed})`,
+      { path: p }
+    );
+  }
+
+  const fluid = parsed.fluid;
+  if (fluid === undefined || fluid === null) return { provider: 'local' };
+  if (typeof fluid !== 'object' || Array.isArray(fluid)) {
+    throw new FluidConfigError(
+      `compose: fluid config at ${p} has a "fluid" key but it is not an object ` +
+      `(got ${Array.isArray(fluid) ? 'array' : typeof fluid})`,
+      { path: p }
+    );
+  }
+  return fluid;
+}
+
+/**
+ * Construct the configured provider.
+ *
+ * @param {string} cwd project root
+ * @param {object} [opts] passed through to the provider's init (the floor takes
+ *   `recordsRoot` to relocate its tracked record tree)
+ * @returns {Promise<import('./provider.js').FluidProvider>}
+ */
+export async function fluidProviderFor(cwd, opts = {}) {
+  return warnIfUnsafelyShared(await construct(cwd, opts));
+}
+
+/**
+ * Warn when a provider's store is reachable from more machines than its
+ * serialization is.
+ *
+ * DERIVED FROM THE PROVIDER'S OWN DECLARATIONS, never hardcoded per provider.
+ * A hardcoded warning has to be remembered by whoever adds the third provider —
+ * which is the same "the second implementation looked complete" failure this
+ * whole feature exists to close — and it has to be remembered AGAIN, in the
+ * other direction, on the day the gap is fixed, or it keeps crying wolf.
+ *
+ * Warns rather than refuses: storage and recall work, COMP-FOH shipped them
+ * deliberately, and refusing would break a configuration that is fine for
+ * everything except concurrent writes. Silence is the only wrong option, because
+ * the failure is invisible until it has cost an idea.
+ */
+function warnIfUnsafelyShared(provider) {
+  if (provider.isShared() && !mutationScopeAtLeast(provider.mutationScope(), MUTATION_SCOPE.CLUSTER)) {
+    console.warn(
+      `compose: fluid provider "${provider.name()}" is shared across machines but serializes `
+      + `mutation only at "${provider.mutationScope()}" scope, so concurrent writes can allocate `
+      + `the same handle and lose records. Prefer the local provider for the ideabox until its `
+      + `store offers a cross-machine reservation primitive (SmartMemory SVC-LEASE-1).`
+    );
+  }
+  return provider;
+}
+
+async function construct(cwd, opts = {}) {
+  const cfg = loadFluidConfig(cwd);
+  const name = cfg.provider ?? 'local';
+
+  if (name === 'local') {
+    return new LocalFluidProvider().init(cwd, { ...cfg.local, ...opts });
+  }
+
+  if (name === 'smartmemory') {
+    // The warning that used to live here is gone, not deleted: it is now
+    // `warnIfUnsafelyShared`, derived from this provider's own
+    // `isShared()`/`mutationScope()` rather than hardcoded to its name
+    // (COMP-FLUID-SEAM-GUARANTEES). BOTH gaps it named are now closed — the
+    // restartable import first, then cross-machine serialization once
+    // SmartMemory shipped SVC-ALLOC-1 and SVC-LEASE-1 (COMP-FLUID-SEAM-GUARANTEES) — so this
+    // provider declares CLUSTER and the warning no longer fires for it. That is
+    // exactly the "remember it AGAIN, in the other direction" the check below
+    // was written to make automatic: nothing here had to be edited to stop the
+    // warning, the declaration moving was enough.
+
+    // The endpoint and credential come from the EXISTING top-level
+    // `smartmemory` block, shared with the shipped kitchen pipeline, so there
+    // is one source of truth for where SmartMemory lives. Only `workspaceId`
+    // is new, and it lives under `fluid.smartmemory` because it scopes fluid
+    // records specifically.
+    //
+    // A missing block is a config error, not a downgrade to the floor: a user
+    // who configured SmartMemory did so to get capabilities the floor does not
+    // have, and starting up on the floor instead would present an
+    // intelligence-free system as a working one.
+    const sm = getSmartmemoryConfig(cwd);
+    if (!sm || Object.keys(sm).length === 0) {
+      throw new FluidConfigError(
+        'compose: fluid provider "smartmemory" requires a top-level "smartmemory" config ' +
+        'block in .compose/compose.json (baseUrl + apiKeyEnv). It is shared with the ' +
+        'SmartMemory ingest pipeline rather than duplicated under "fluid".',
+        { provider: name, setting: 'smartmemory' }
+      );
+    }
+    return new SmartMemoryFluidProvider().init(cwd, {
+      baseUrl: sm.baseUrl,
+      apiKeyEnv: sm.apiKeyEnv,
+      timeoutMs: sm.timeoutMs,
+      ...cfg.smartmemory,
+      ...opts,
+    });
+  }
+
+  throw new FluidConfigError(`compose: unknown fluid provider "${name}"`, { provider: name });
+}
+
+export { loadFluidConfig };

package/lib/fluid/ideabox-dates.js

@@ -0,0 +1,73 @@
+/**
+ * lib/fluid/ideabox-dates.js — the date vocabulary of the markdown boundary.
+ *
+ * COMP-PLAN-IDEA-UNIFY S3b-1.
+ *
+ * The record contract types every event date as `format: "date-time"`
+ * (`contracts/fluid-record.schema.json` — `killed.at`, `discussion[].at`), but
+ * the ideabox markdown has only ever carried a bare `YYYY-MM-DD`: the
+ * discussion grammar is `- [YYYY-MM-DD] author: text`
+ * (`lib/ideabox.js:53`) and `killIdea` stamps `…toISOString().slice(0, 10)`
+ * (`lib/ideabox.js:546`).
+ *
+ * Two directions, and BOTH were wrong before this module existed:
+ *
+ *   import  markdown date  → record date-time   (widen)
+ *   render  record date-time → markdown date    (narrow)
+ *
+ * The importer passed the bare date straight into the contract, so importing
+ * any idea carrying a discussion entry or a kill date threw
+ * `must match format "date-time"`. The renderer emitted the full ISO timestamp
+ * straight into the markdown, where `DISCUSSION_ENTRY_RE` cannot match it — so
+ * a provider-written discussion entry degraded to an unparsed extra line and
+ * was silently lost on the next read.
+ *
+ * Neither defect was caught, because no idea on disk had ever carried a
+ * discussion entry and the Killed Ideas section was empty. Both paths were
+ * dead code with passing tests over them.
+ *
+ * They live together in one module deliberately: the two conversions are a
+ * single round-trip contract, and their drifting apart is precisely the bug.
+ * Splitting them across the importer and the renderer is what let it happen.
+ *
+ * Precision is intentionally asymmetric. A record keeps the full timestamp; the
+ * projection is a view and shows the day. Rendering a date-time into a file
+ * whose grammar is a date does not preserve information, it corrupts the line.
+ */
+
+/** A bare calendar date, the only date form the ideabox markdown can carry. */
+const MARKDOWN_DATE_RE = /^\d{4}-\d{2}-\d{2}$/;
+
+const EPOCH = new Date(0).toISOString();
+
+/**
+ * Widen a markdown date to a contract-valid `date-time`.
+ *
+ * A value that already carries a time is passed through untouched, so this is
+ * safe to apply to input of mixed provenance (an imported entry and a
+ * provider-written one can sit in the same array).
+ *
+ * @param {string|null|undefined} value a `YYYY-MM-DD`, a full ISO timestamp, or nothing
+ * @returns {string} an ISO 8601 date-time
+ */
+export function toRecordTimestamp(value) {
+  if (!value) return EPOCH;
+  if (MARKDOWN_DATE_RE.test(value)) return `${value}T00:00:00.000Z`;
+  return value;
+}
+
+/**
+ * Narrow a contract `date-time` to the markdown's `YYYY-MM-DD`.
+ *
+ * Anything unparseable is returned unchanged rather than coerced: emitting a
+ * wrong-but-well-formed date would be worse than emitting the raw value, which
+ * is at least visibly odd.
+ *
+ * @param {string|null|undefined} value an ISO 8601 date-time
+ * @returns {string} a `YYYY-MM-DD`
+ */
+export function toMarkdownDate(value) {
+  if (typeof value !== 'string') return '';
+  const head = value.slice(0, 10);
+  return MARKDOWN_DATE_RE.test(head) ? head : value;
+}

package/lib/fluid/ideabox-migrate.js

@@ -0,0 +1,154 @@
+/**
+ * lib/fluid/ideabox-migrate.js — the first-use gate between a markdown ideabox
+ * and the record store.
+ *
+ * COMP-PLAN-IDEA-UNIFY S3b-1 (F2).
+ *
+ * THE FAILURE THIS EXISTS TO PREVENT
+ * ----------------------------------
+ * `@smartmemory/compose` is published. Other projects have their own populated
+ * `docs/product/ideabox.md` and no fluid records. The cutover makes records
+ * canon and the markdown a projection of them — so without this gate, the first
+ * `compose ideabox add` in an upgraded project allocates IDEA-1 against an empty
+ * store and the projection replaces that project's entire ideabox with the one
+ * idea they just typed. Their ideas would survive only in their git history.
+ *
+ * The blueprint missed this by treating the import as a one-time operation on
+ * THIS repository. It is not: it is an upgrade path, and it runs once per
+ * installation, on whatever that installation happens to have.
+ *
+ * WHY IT REFUSES RATHER THAN REPAIRS
+ * ----------------------------------
+ * The dangerous state is not "no records" — that one is unambiguous and is
+ * simply imported. It is a PARTIAL store: some records present, and markdown
+ * entries that have no record behind them. Any automatic reading of that state
+ * is a guess. Importing the strays assumes the markdown is authoritative, which
+ * it no longer is. Ignoring them assumes they were deliberately deleted, and
+ * projects over them. Both silently discard someone's work in one of the two
+ * cases. So it stops and names what it found.
+ *
+ * That check costs a parse of a small file per mutation, which is the same file
+ * the CLI already read on every mutation before the cutover.
+ */
+
+import { existsSync, readFileSync } from 'node:fs';
+
+import { parseIdeabox } from '../ideabox.js';
+import { importIdeabox } from './import-ideabox.js';
+import { KIND } from './provider.js';
+
+export class IdeaboxMigrationConflict extends Error {
+  constructor(missing, ideaboxPath) {
+    super(
+      `compose: the ideabox at ${ideaboxPath} contains ${missing.length} idea(s) with no record ` +
+      `behind them: ${missing.join(', ')}. The record store is canon now, so this file is ` +
+      `generated output — which means these entries were either hand-added after the migration ` +
+      `or lost by a partial one, and guessing which would discard someone's work either way. ` +
+      `Re-add them with \`compose ideabox add\`, or delete them from the file if they are stale, ` +
+      `then run \`compose ideabox render\`.`
+    );
+    this.name = 'IdeaboxMigrationConflict';
+    this.code = 'IDEABOX_MIGRATION_CONFLICT';
+    this.missing = missing;
+  }
+}
+
+/**
+ * Ensure the record store reflects the markdown before any mutation touches it.
+ *
+ * Runs before every mutating ideabox command. Three states, one of which stops
+ * the command:
+ *
+ *   - **no records, markdown has entries** → import it (the upgrade path)
+ *   - **records exist, markdown adds nothing** → already migrated, proceed
+ *   - **records exist, markdown has entries with no record** → refuse
+ *
+ * A fresh project with no markdown and no records is the trivial first case and
+ * simply proceeds.
+ *
+ * @param {import('./provider.js').FluidProvider} provider
+ * @param {string} ideaboxPath absolute path to the markdown ideabox
+ * @returns {Promise<{migrated: boolean, imported: string[]}>}
+ */
+export async function ensureIdeaboxMigrated(provider, ideaboxPath) {
+  const records = await provider.listRecords({ kind: KIND.IDEA });
+  const markdown = existsSync(ideaboxPath) ? readFileSync(ideaboxPath, 'utf8') : null;
+
+  if (markdown === null) return { migrated: false, imported: [] };
+
+  const parsed = parseIdeabox(markdown);
+  const inMarkdown = [...(parsed.ideas ?? []), ...(parsed.killed ?? [])].map((i) => i.id);
+
+  const known = new Set(records.map((r) => r.handle));
+  const missing = inMarkdown.filter((id) => !known.has(id));
+
+  if (records.length === 0) {
+    if (inMarkdown.length === 0) return { migrated: false, imported: [] };
+    // The upgrade path.
+    const result = await importIdeabox(provider, { markdown, path: ideaboxPath });
+    return { migrated: true, imported: result.imported };
+  }
+
+  if (missing.length) {
+    // RESUME versus REFUSE, decided PER HANDLE from the events log.
+    //
+    // A crash partway through the first-use import leaves some records written
+    // and the rest missing, which lands here rather than in the empty-store
+    // branch above. Refusing that outright strands the installation: the error
+    // names `compose ideabox add`, `add` runs this same gate, so every command
+    // fails and there is no way out — and the reclaim path built for exactly
+    // this case is never reached.
+    //
+    // The log distinguishes the two populations. A handle the import already
+    // burned carries an event; `importIdeabox` skips live records and reclaims
+    // its own aborted allocations, so resuming is safe and lossless.
+    //
+    // The evidence has to be per-handle, not "did an import ever run". Once the
+    // first import succeeds the log carries `imported` events forever, so a
+    // global check would quietly import anything later hand-added to what is
+    // now generated output — losing the very protection this gate exists for.
+    // A handle with no event was never issued here: it was typed into the file
+    // by hand, and importing it would treat the markdown as authoritative when
+    // it no longer is.
+    // Three populations, and only one of them is resumable:
+    //   - issued, no `deleted` event  → a create that crashed. RESUME.
+    //   - issued, `deleted` event     → deliberately retired; this file is just
+    //                                   stale output. REFUSE (a render fixes it,
+    //                                   and importing would resurrect it).
+    //   - never issued               → hand-typed into generated output. REFUSE.
+    const { issued, deleted } = await handleHistory(provider);
+    const resumable = missing.filter((id) => issued.has(id) && !deleted.has(id));
+    const strays = missing.filter((id) => !resumable.includes(id));
+    if (strays.length) throw new IdeaboxMigrationConflict(strays, ideaboxPath);
+
+    const result = await importIdeabox(provider, { markdown, path: ideaboxPath });
+    return { migrated: true, imported: result.imported };
+  }
+
+  return { migrated: false, imported: [] };
+}
+
+/**
+ * Which handles this store's log has seen, and which of those were retired.
+ *
+ * Both sets are empty when the history cannot be read, which makes every
+ * missing handle a stray and every ambiguous state a refusal. That is the safe
+ * direction: resuming on a guess is the one thing this must not do.
+ */
+async function handleHistory(provider) {
+  const empty = { issued: new Set(), deleted: new Set() };
+  if (typeof provider.readEvents !== 'function') return empty;
+  try {
+    const events = (await provider.readEvents()) ?? [];
+    const issued = new Set();
+    const deleted = new Set();
+    for (const event of events) {
+      if (!event?.handle) continue;
+      issued.add(event.handle);
+      if (event.type === 'deleted') deleted.add(event.handle);
+    }
+    return { issued, deleted };
+  } catch {
+    return empty;
+  }
+}