@ngockhoale/ukit 2.7.13 → 2.8.0

package/src/core/memory/recordStore.js

@@ -0,0 +1,81 @@
+// Record-document store core — SPEC §3 / FR-007.
+// Path-agnostic read/write of the v2 records document
+// ({ schemaVersion: 2, records: [...] }) so both the project store
+// (storeV2.js) and the user store (userMemory.js) share one implementation.
+// Writes go through fileOps.writeJson (tmp+rename).
+
+import { readJsonIfExists, writeJson } from '../fileOps.js';
+import { normalizeRecord } from './records.js';
+
+const SCHEMA_VERSION = 2;
+
+export const PATCHABLE_FIELDS = ['type', 'scope', 'text', 'provenance', 'confidence',
+  'valid_until', 'status', 'project_id', 'meta', 'source_fingerprint'];
+
+export function isRawRecord(input) {
+  return input && typeof input === 'object' && typeof input.id === 'string'
+    && typeof input.status === 'string';
+}
+
+/**
+ * readRecordStore(recordsPath) → { records, invalidSkipped }
+ * Tolerant: missing file → {records:[]}; unparseable → {records:[], invalidSkipped:1};
+ * per-record normalizeRecord skip.
+ */
+export async function readRecordStore(recordsPath) {
+  let doc;
+  try {
+    doc = await readJsonIfExists(recordsPath);
+  } catch {
+    return { records: [], invalidSkipped: 1 };
+  }
+  if (!doc) return { records: [], invalidSkipped: 0 };
+
+  const rawRecords = Array.isArray(doc.records) ? doc.records : [];
+  const records = [];
+  let invalidSkipped = Array.isArray(doc.records) ? 0 : 1;
+  for (const raw of rawRecords) {
+    const normalized = normalizeRecord(raw);
+    if (normalized) records.push(normalized);
+    else invalidSkipped += 1;
+  }
+  return { records, invalidSkipped };
+}
+
+/**
+ * writeRecordStore(recordsPath, records) → void
+ * Atomic write; drops unrecoverable entries.
+ */
+export async function writeRecordStore(recordsPath, records) {
+  const normalized = (Array.isArray(records) ? records : [])
+    .map((raw) => normalizeRecord(raw))
+    .filter(Boolean);
+  await writeJson(recordsPath, {
+    schemaVersion: SCHEMA_VERSION,
+    records: normalized,
+  });
+}
+
+/**
+ * applyRecordPatch(records, id, patch) → { record, records } | null
+ * Patches PATCHABLE_FIELDS on the record with `id`, re-normalizes, and returns
+ * the updated list (caller persists). Throws when the patch is unrecoverable.
+ */
+export function applyRecordPatch(records, id, patch = {}) {
+  const index = records.findIndex((r) => r.id === id);
+  if (index === -1) return null;
+
+  const next = { ...records[index] };
+  for (const field of PATCHABLE_FIELDS) {
+    if (Object.prototype.hasOwnProperty.call(patch, field)) {
+      next[field] = patch[field];
+    }
+  }
+  const normalized = normalizeRecord(next);
+  if (!normalized) {
+    throw new Error(`applyRecordPatch: patch produces an invalid record (id ${id})`);
+  }
+  const updated = [...records];
+  updated[index] = normalized;
+  return { record: normalized, records: updated };
+}

package/src/core/memory/storeV2.js

@@ -4,14 +4,19 @@
 // Read ops call ensureMigrated() first so every consumer triggers the lazy
 // v1→v2 migration (SPEC §4); migrate.js is lazy-imported to break the
 // storeV2↔migrate import cycle (runMigration writes via saveRecords).
+// The record-document read/write core lives in recordStore.js (FR-007) so the
+// user-level store (userMemory.js) shares it without the migration flow.
 
 import fs from 'node:fs/promises';
 import { buildRuntimePaths } from '../runtimePaths.js';
-import { readJsonIfExists, writeJson } from '../fileOps.js';
 import { loadRuntimeConfig } from '../runtimeConfig.js';
 import { createRecord, normalizeRecord } from './records.js';
-
-const SCHEMA_VERSION = 2;
+import {
+  readRecordStore,
+  writeRecordStore,
+  applyRecordPatch,
+  isRawRecord,
+} from './recordStore.js';
 
 // Per-process memoization for ensureMigrated — one-shot check per projectRoot.
 const migratedRoots = new Set();
@@ -43,11 +48,6 @@ async function ensureMigrated(projectRoot) {
   }
 }
 
-function isRawRecord(input) {
-  return input && typeof input === 'object' && typeof input.id === 'string'
-    && typeof input.status === 'string';
-}
-
 /**
  * loadRecords(projectRoot) → record[] — tolerant: invalid entries skipped.
  */
@@ -59,23 +59,7 @@ export async function loadRecords(projectRoot) {
 
 async function readStore(projectRoot) {
   const paths = buildRuntimePaths(projectRoot);
-  let doc;
-  try {
-    doc = await readJsonIfExists(paths.memoryV2RecordsPath);
-  } catch {
-    return { records: [], invalidSkipped: 1 };
-  }
-  if (!doc) return { records: [], invalidSkipped: 0 };
-
-  const rawRecords = Array.isArray(doc.records) ? doc.records : [];
-  const records = [];
-  let invalidSkipped = Array.isArray(doc.records) ? 0 : 1;
-  for (const raw of rawRecords) {
-    const normalized = normalizeRecord(raw);
-    if (normalized) records.push(normalized);
-    else invalidSkipped += 1;
-  }
-  return { records, invalidSkipped };
+  return readRecordStore(paths.memoryV2RecordsPath);
 }
 
 /**
@@ -83,13 +67,7 @@ async function readStore(projectRoot) {
  */
 export async function saveRecords(projectRoot, records) {
   const paths = buildRuntimePaths(projectRoot);
-  const normalized = (Array.isArray(records) ? records : [])
-    .map((raw) => normalizeRecord(raw))
-    .filter(Boolean);
-  await writeJson(paths.memoryV2RecordsPath, {
-    schemaVersion: SCHEMA_VERSION,
-    records: normalized,
-  });
+  await writeRecordStore(paths.memoryV2RecordsPath, records);
 }
 
 /**
@@ -108,31 +86,17 @@ export async function addRecord(projectRoot, recordInput) {
   return record;
 }
 
-const PATCHABLE_FIELDS = ['type', 'scope', 'text', 'provenance', 'confidence',
-  'valid_until', 'status', 'project_id', 'meta', 'source_fingerprint'];
-
 /**
  * updateRecord(projectRoot, id, patch) → record | null
  */
 export async function updateRecord(projectRoot, id, patch = {}) {
   await ensureMigrated(projectRoot);
-  const { records } = await readStore(projectRoot);
-  const index = records.findIndex((r) => r.id === id);
-  if (index === -1) return null;
-
-  const next = { ...records[index] };
-  for (const field of PATCHABLE_FIELDS) {
-    if (Object.prototype.hasOwnProperty.call(patch, field)) {
-      next[field] = patch[field];
-    }
-  }
-  const normalized = normalizeRecord(next);
-  if (!normalized) {
-    throw new Error(`updateRecord: patch produces an invalid record (id ${id})`);
-  }
-  records[index] = normalized;
-  await saveRecords(projectRoot, records);
-  return normalized;
+  const paths = buildRuntimePaths(projectRoot);
+  const { records } = await readRecordStore(paths.memoryV2RecordsPath);
+  const result = applyRecordPatch(records, id, patch);
+  if (!result) return null;
+  await writeRecordStore(paths.memoryV2RecordsPath, result.records);
+  return result.record;
 }
 
 /**

package/src/core/memory/userMemory.js

@@ -0,0 +1,111 @@
+// User-scope memory store — SPEC FR-008.
+// Canonical user-level memory API at ~/.ukit/storage/memory/v2/records.json,
+// shared across projects. Same v2 record document as the project store but
+// with NO v1→v2 migration (the user layer starts at v2). All functions take
+// `{homeDir}={}` and resolve paths via buildUserPaths.
+
+import { buildUserPaths } from '../userPaths.js';
+import { createRecord, normalizeRecord } from './records.js';
+import {
+  readRecordStore,
+  writeRecordStore,
+  applyRecordPatch,
+  isRawRecord,
+} from './recordStore.js';
+import { loadRecords } from './storeV2.js';
+
+function userRecordsPath(homeDir) {
+  return buildUserPaths({ homeDir }).memoryV2RecordsPath;
+}
+
+/**
+ * loadUserRecords({homeDir}={}) → record[] — tolerant: invalid entries skipped.
+ */
+export async function loadUserRecords({ homeDir } = {}) {
+  const { records } = await readRecordStore(userRecordsPath(homeDir));
+  return records;
+}
+
+/**
+ * saveUserRecords(records, {homeDir}={}) → void — atomic write.
+ */
+export async function saveUserRecords(records, { homeDir } = {}) {
+  await writeRecordStore(userRecordsPath(homeDir), records);
+}
+
+/**
+ * addUserRecord(recordInput, {homeDir}={}) → record
+ * Accepts either createRecord-style input or a full raw record (normalized).
+ */
+export async function addUserRecord(recordInput, { homeDir } = {}) {
+  const record = isRawRecord(recordInput)
+    ? normalizeRecord(recordInput)
+    : createRecord(recordInput);
+  if (!record) {
+    throw new Error('addUserRecord: input could not be normalized into a valid record');
+  }
+  const recordsPath = userRecordsPath(homeDir);
+  const { records } = await readRecordStore(recordsPath);
+  await writeRecordStore(recordsPath, [...records, record]);
+  return record;
+}
+
+/**
+ * updateUserRecord(id, patch, {homeDir}={}) → record | null
+ */
+export async function updateUserRecord(id, patch = {}, { homeDir } = {}) {
+  const recordsPath = userRecordsPath(homeDir);
+  const { records } = await readRecordStore(recordsPath);
+  const result = applyRecordPatch(records, id, patch);
+  if (!result) return null;
+  await writeRecordStore(recordsPath, result.records);
+  return result.record;
+}
+
+/**
+ * getUserRecord(id, {homeDir}={}) → record | null
+ */
+export async function getUserRecord(id, { homeDir } = {}) {
+  const { records } = await readRecordStore(userRecordsPath(homeDir));
+  return records.find((r) => r.id === id) ?? null;
+}
+
+/**
+ * queryUserRecords({ type?, scope?, status?, projectId? }, {homeDir}={}) → record[]
+ */
+export async function queryUserRecords({ type, scope, status, projectId } = {}, { homeDir } = {}) {
+  const records = await loadUserRecords({ homeDir });
+  return records.filter((r) => (type == null || r.type === type)
+    && (scope == null || r.scope === scope)
+    && (status == null || r.status === status)
+    && (projectId == null || r.project_id === projectId));
+}
+
+/**
+ * userMemoryStats({homeDir}={}) → { total, byType, byStatus, invalidSkipped }
+ */
+export async function userMemoryStats({ homeDir } = {}) {
+  const { records, invalidSkipped } = await readRecordStore(userRecordsPath(homeDir));
+  const byType = {};
+  const byStatus = {};
+  for (const r of records) {
+    byType[r.type] = (byType[r.type] ?? 0) + 1;
+    byStatus[r.status] = (byStatus[r.status] ?? 0) + 1;
+  }
+  return { total: records.length, byType, byStatus, invalidSkipped };
+}
+
+/**
+ * loadMergedRecords(projectRoot, {homeDir}={}) → record[]
+ * Project records ++ user records; the user record wins on id collision.
+ */
+export async function loadMergedRecords(projectRoot, { homeDir } = {}) {
+  const [projectRecords, userRecords] = await Promise.all([
+    loadRecords(projectRoot),
+    loadUserRecords({ homeDir }),
+  ]);
+  const merged = new Map();
+  for (const r of projectRecords) merged.set(r.id, r);
+  for (const r of userRecords) merged.set(r.id, r);
+  return [...merged.values()];
+}

package/src/core/paths.js

@@ -5,6 +5,7 @@ export function buildPathConfig({ packageRoot, projectRoot }) {
     packageRoot,
     projectRoot,
     manifestPath: path.join(packageRoot, 'manifests', 'platform.full.yaml'),
+    userManifestPath: path.join(packageRoot, 'manifests', 'platform.user.yaml'),
     templatesRoot: path.join(packageRoot, 'templates'),
     ukitRoot: path.join(projectRoot, '.claude', 'ukit', '.ukit'),
     installMetaPath: path.join(projectRoot, '.claude', 'ukit', '.ukit', 'install.json'),

package/src/core/runInstallPipeline.js

@@ -1,4 +1,5 @@
 import fs from 'node:fs/promises';
+import os from 'node:os';
 import path from 'node:path';
 import { loadManifest } from '../manifest/loadManifest.js';
 import { detectStack } from '../stack/detectStack.js';
@@ -14,9 +15,10 @@ import { cleanupLegacyPaths, migrateLegacyRuntimeRoot } from './migrateLegacy.js
 import { ensureGitignore } from './ensureGitignore.js';
 import { repairBrokenHooks } from './repairBrokenHooks.js';
 import { applyGatewayResilienceEnv } from './gatewayResilienceEnv.js';
-import { cleanupEmptyParents, readJsonIfExists, removeFileOrLinkOnly, resolveProjectRelativePath } from './fileOps.js';
+import { cleanupEmptyParents, readJsonIfExists, removeFileOrLinkOnly, resolveProjectRelativePath, writeJson } from './fileOps.js';
 import { findOpencodeArtifacts, opencodeSteerMessage } from './opencodeSteer.js';
 import { loadRuntimeConfig } from './runtimeConfig.js';
+import { buildUserPaths } from './userPaths.js';
 
 const AUTO_PRUNE_OBSOLETE_PREFIXES = [
   '.claude/skills/',
@@ -247,6 +249,74 @@ async function pruneObsoleteManagedPaths({
 
   return removedCount;
 }
+// SPEC FR-004 — user-layer seeding pass. Runs after the project plan/diff (and
+// after the project apply on real installs): seeds ~/.ukit/ from
+// manifests/platform.user.yaml. Every user item is `mergeStrategy: skip`, so
+// existing user files are never touched — install only creates what is missing.
+// The pass is advisory: any failure is caught, warned, and reported via
+// `userLayer.error` — it must never fail the project install.
+// User-layer writes are deliberately NOT fed to writeInstallMetadata or
+// buildManagedRelativePathSet — those are project-scoped; the user layer has its
+// own marker at ~/.ukit/install.json.
+async function runUserLayerPass({
+  pathConfig,
+  homeDir,
+  stackContext,
+  variables,
+  selectedAdapterItemIds,
+  packageVersion,
+  dryRun,
+}) {
+  const userLayer = { seeded: 0, kept: 0, skipped: 0, error: null, userRoot: null };
+  let userRows = [];
+  try {
+    // buildUserPaths lives inside the try so a bad homeDir can never fail the
+    // project install — the whole pass is advisory.
+    const userPaths = buildUserPaths({ homeDir });
+    userLayer.userRoot = userPaths.userRoot;
+    // pathConfig.userManifestPath is set by buildPathConfig; fall back to the
+    // package layout (templatesRoot's sibling manifests/) for callers that build
+    // pathConfig by hand (tests, embedded runners).
+    const userManifestPath = pathConfig.userManifestPath
+      ?? path.join(pathConfig.templatesRoot, '..', 'manifests', 'platform.user.yaml');
+    const userManifest = await loadManifest(userManifestPath);
+    const userPlan = await buildInstallPlan({
+      manifest: userManifest,
+      stackContext,
+      templatesRoot: pathConfig.templatesRoot,
+      variables,
+      projectRoot: userPaths.userRoot,
+      selectedAdapterItemIds,
+    });
+    const userDiff = await diffInstallPlan(userPlan);
+    const userSummary = summarizeDiff(userDiff);
+    userRows = toDiffRows(userDiff).map((row) => ({ ...row, layer: 'user' }));
+
+    if (!dryRun) {
+      const { skippedByAction } = await applyDiffResults(userDiff, {
+        backupRoot: path.join(userPaths.userRoot, 'backups'),
+        projectRoot: userPaths.userRoot,
+      });
+      userLayer.seeded = userSummary.create - (skippedByAction?.create ?? 0);
+      userLayer.kept = userSummary.skip + userSummary.unchanged;
+      userLayer.skipped = (skippedByAction?.create ?? 0) + (skippedByAction?.update ?? 0);
+      // UKit-managed marker (not user data): records that the pass ran.
+      await writeJson(userPaths.installMetaPath, {
+        tool: 'ukit',
+        version: packageVersion,
+        seededAt: new Date().toISOString(),
+      });
+    } else {
+      userLayer.seeded = userSummary.create;
+      userLayer.kept = userSummary.skip + userSummary.unchanged;
+    }
+  } catch (error) {
+    userLayer.error = error?.message ?? String(error);
+    console.warn(`[UKit] Warning: user-layer seeding skipped — ${userLayer.error}`);
+  }
+  return { userLayer, userRows };
+}
+
 
 export async function runInstallPipeline({
   packageVersion,
@@ -255,6 +325,7 @@ export async function runInstallPipeline({
   selectedAdapterItemIds,
   retainedManagedRelativePaths = [],
   withCodegraph = false,
+  homeDir = os.homedir(),
 }) {
   // Check for an existing install before making any changes.
   // cleanupLegacyPaths() is only safe to run on reinstalls: on a fresh install
@@ -404,28 +475,50 @@ export async function runInstallPipeline({
       retainedManagedRelativePaths,
     });
 
+    const { userLayer, userRows } = await runUserLayerPass({
+      pathConfig,
+      homeDir,
+      stackContext,
+      variables,
+      selectedAdapterItemIds,
+      packageVersion,
+      dryRun: false,
+    });
+
     return {
       manifest,
       stackContext,
       projectContext,
       providerContext,
       summary,
-      rows: toDiffRows(diffResults),
+      rows: [...toDiffRows(diffResults), ...userRows],
       writes,
       hookRepair,
       gatewayResilience,
+      userLayer,
     };
   }
 
+  const { userLayer, userRows } = await runUserLayerPass({
+    pathConfig,
+    homeDir,
+    stackContext,
+    variables,
+    selectedAdapterItemIds,
+    packageVersion,
+    dryRun: true,
+  });
+
   return {
     manifest,
     stackContext,
     projectContext,
     providerContext,
     summary: plannedSummary,
-    rows: toDiffRows(diffResults),
+    rows: [...toDiffRows(diffResults), ...userRows],
     writes: [],
     hookRepair: { removals: [], files: [] },
     gatewayResilience: { customGateway: false, reason: 'dry-run' },
+    userLayer,
   };
 }