@smartmemory/compose 0.1.34-beta → 0.1.35-beta

package/README.md

@@ -82,6 +82,48 @@ Check what you're running:
 compose --version
 ```
 
+## Tracker providers
+
+Compose can persist feature data to different backends via the `tracker` block in `.compose/compose.json`.
+
+**Default (local) — zero configuration required:**
+
+```json
+{ "tracker": { "provider": "local" } }
+```
+
+`local` is the default when no `tracker` block is present. All writes go to the filesystem exactly as before — no behavior change.
+
+**GitHub provider:**
+
+```json
+{
+  "tracker": {
+    "provider": "github",
+    "github": {
+      "repo": "owner/repo",
+      "projectNumber": 42,
+      "branch": "main",
+      "roadmapPath": "ROADMAP.md",
+      "changelogPath": "CHANGELOG.md",
+      "cacheTtlSeconds": 300,
+      "auth": { "tokenEnv": "GITHUB_TOKEN" }
+    }
+  }
+}
+```
+
+The GitHub provider syncs features to **Issues** (one per feature), **Projects v2** (`Status` custom field), and **Contents API** (roadmap + changelog files). Requires a token in the named env var (or `gh auth login` fallback) with `repo` and `project` scopes.
+
+CLI verbs:
+
+```bash
+compose tracker status   # show provider health + pending op-log + conflict ledger
+compose tracker sync     # reconcile op-log against remote provider
+```
+
+See [docs/configuration.md](docs/configuration.md) for the full `tracker` config reference.
+
 ## Documentation
 
 Topic-scoped reference:

package/bin/compose.js

@@ -2674,6 +2674,22 @@ if (cmd === 'build') {
     process.exit(1)
   }
 
+} else if (cmd === 'tracker') {
+  // ---------------------------------------------------------------------------
+  // compose tracker status   — print provider name, canonical, pendingOps, conflicts, mixedSources
+  // compose tracker sync     — flush op-log and report drained/quarantined counts
+  // COMP-TRACKER-PROVIDER T18
+  // ---------------------------------------------------------------------------
+  try {
+    const { runTrackerCli } = await import('../lib/tracker/cli.js')
+    const result = await runTrackerCli(process.cwd(), args)
+    console.log(result.output)
+    if (result.exitCode !== 0) process.exit(result.exitCode)
+  } catch (err) {
+    console.error(`tracker: ${err.message}`)
+    process.exit(1)
+  }
+
 } else {
   console.error(`Unknown command: ${cmd}`)
   process.exit(1)

package/lib/build.js

@@ -31,8 +31,16 @@ import { emitSections as emitPlanSections, appendTrailers as appendSectionTraile
 import { SECTIONS_DIR } from './constants.js';
 
 import YAML from 'yaml';
-import { updateFeature, readFeature, writeFeature } from './feature-json.js';
+// feature-json direct imports removed — mutations now go through TrackerProvider (T9)
 import { loadFeaturesDir } from './project-paths.js';
+
+// Lazy provider accessor — avoids circular import risk (factory → local-provider
+// does NOT import build.js, so a static import is safe, but lazy is used for
+// consistency with the pattern established in T7/T8 and to avoid any future risk).
+async function getBuildProvider(cwd) {
+  const { providerFor } = await import('./tracker/factory.js');
+  return providerFor(cwd);
+}
 import { evaluatePolicy } from '../server/policy-evaluator.js';
 import { runTriage, isTriageStale } from './triage.js';
 import { shouldRunCrossModel, LENS_DEFINITIONS } from './review-lenses.js';
@@ -622,18 +630,22 @@ export async function runBuild(featureCode, opts = {}) {
   //   - opts.template is explicitly set (user chose a specific template)
   // ---------------------------------------------------------------------------
   let buildProfile = null;
+  let _buildTierLabel = '?'; // for skip_reason label in spec YAML mutation below
   // Bug mode skips pre-build triage entirely — triage is feature-shaped
   // (writes feature.json, profile selection per feature complexity tiers).
   if (!isBugMode && !opts.skipTriage && !opts.template) {
-    let cachedFeature = readFeature(cwd, featureCode, featuresDir);
+    const _buildProvider = await getBuildProvider(cwd);
+    let cachedFeature = await _buildProvider.getFeature(featureCode);
     if (cachedFeature?.profile && !isTriageStale(cwd, featureCode, featuresDir)) {
       // Reuse cached profile
       buildProfile = cachedFeature.profile;
-      console.log(`[triage] Using cached profile (tier ${cachedFeature.complexity ?? '?'}): ${JSON.stringify(buildProfile)}`);
+      _buildTierLabel = cachedFeature.complexity ?? '?';
+      console.log(`[triage] Using cached profile (tier ${_buildTierLabel}): ${JSON.stringify(buildProfile)}`);
     } else {
       // Run fresh triage
       const triageResult = await runTriage(featureCode, { cwd, featuresDir });
       buildProfile = triageResult.profile;
+      _buildTierLabel = String(triageResult.tier);
       console.log(`[triage] Tier ${triageResult.tier}: ${triageResult.rationale}`);
       console.log(`[triage] Profile: ${JSON.stringify(buildProfile)}`);
 
@@ -641,20 +653,23 @@ export async function runBuild(featureCode, opts = {}) {
       if (!cachedFeature) {
         // Create feature.json — feature folder exists but json was missing
         const featureDesc = opts.description ?? featureCode;
-        writeFeature(cwd, {
+        cachedFeature = await _buildProvider.createFeature(featureCode, {
           code: featureCode,
           description: featureDesc,
           status: 'PLANNED',
           complexity: String(triageResult.tier),
           profile: buildProfile,
           triageTimestamp,
-        }, featuresDir);
+        });
       } else {
-        updateFeature(cwd, featureCode, {
+        // Profile/complexity cache update — no status change. Spread current
+        // feature so putFeature receives the full object (it overwrites, not merges).
+        cachedFeature = await _buildProvider.putFeature(featureCode, {
+          ...cachedFeature,
           complexity: String(triageResult.tier),
           profile: buildProfile,
           triageTimestamp,
-        }, featuresDir);
+        });
       }
     }
   }
@@ -689,9 +704,8 @@ export async function runBuild(featureCode, opts = {}) {
           delete step.skip_reason;
         } else if (buildProfile[needsKey] === false) {
           // Disable step — mark as unconditionally skipped
-          const tier = readFeature(cwd, featureCode, featuresDir)?.complexity ?? '?';
           step.skip_if = 'true';
-          step.skip_reason = `Skipped by triage (tier ${tier})`;
+          step.skip_reason = `Skipped by triage (tier ${_buildTierLabel})`;
         }
       }
       specYaml = YAML.stringify(specObj);
@@ -752,7 +766,16 @@ export async function runBuild(featureCode, opts = {}) {
   // Update feature.json status to IN_PROGRESS (feature mode only;
   // bug mode does not use feature.json).
   if (!isBugMode) {
-    updateFeature(cwd, featureCode, { status: 'IN_PROGRESS' }, featuresDir);
+    const _bp = await getBuildProvider(cwd);
+    // Guard: feature.json may not exist if triage was skipped AND no prior
+    // createFeature ran (e.g. test harnesses that only create the folder).
+    // Original updateFeature silently no-oped when feature was missing.
+    // Use persistFeatureRaw (not setStatus) — raw write with no transition policy,
+    // no events, no renderRoadmap. Matches original updateFeature semantics exactly.
+    const _feat = await _bp.getFeature(featureCode);
+    if (_feat) {
+      await _bp.persistFeatureRaw(featureCode, { ..._feat, status: 'IN_PROGRESS' });
+    }
   }
 
   // Hoisted for finally-block visibility
@@ -1831,7 +1854,15 @@ export async function runBuild(featureCode, opts = {}) {
       // COMP-QA: persist filesChanged so `compose qa-scope` can read them post-build.
       // Bug mode skips feature-json — bugs don't have feature.json (COMP-FIX-HARD T4).
       if (!isBugMode) {
-        updateFeature(cwd, featureCode, { status: 'COMPLETE', filesChanged: context.filesChanged ?? [] }, featuresDir);
+        const _bp = await getBuildProvider(cwd);
+        // Guard: feature.json may not exist when triage was skipped (test harnesses).
+        // Original updateFeature silently no-oped when feature was missing.
+        // Single atomic raw write (status + filesChanged together) — restores original
+        // updateFeature atomicity. persistFeatureRaw: no policy, no events, no roadmap.
+        const _feat = await _bp.getFeature(featureCode);
+        if (_feat) {
+          await _bp.persistFeatureRaw(featureCode, { ..._feat, status: 'COMPLETE', filesChanged: context.filesChanged ?? [] });
+        }
       }
       const termState = readActiveBuild(dataDir);
       if (termState) {
@@ -1842,7 +1873,15 @@ export async function runBuild(featureCode, opts = {}) {
       buildStatus = 'killed';
       console.log('\nBuild killed.');
       await visionWriter.updateItemStatus(itemId, 'killed');
-      if (!isBugMode) updateFeature(cwd, featureCode, { status: 'PLANNED' }, featuresDir);
+      if (!isBugMode) {
+        const _bp = await getBuildProvider(cwd);
+        const _feat = await _bp.getFeature(featureCode);
+        if (_feat) {
+          // Raw write back to PLANNED — no transition policy, no events, no renderRoadmap.
+          // Matches original updateFeature semantics; keeps teardown side-effect-free.
+          await _bp.persistFeatureRaw(featureCode, { ..._feat, status: 'PLANNED' });
+        }
+      }
       const termState = readActiveBuild(dataDir);
       if (termState) {
         writeActiveBuild(dataDir, { ...termState, status: 'aborted', completedAt: new Date().toISOString() });
@@ -1851,7 +1890,15 @@ export async function runBuild(featureCode, opts = {}) {
       // Ship failure or other explicit failure — write terminal state
       console.log('\nBuild failed.');
       await visionWriter.updateItemStatus(itemId, 'failed');
-      if (!isBugMode) updateFeature(cwd, featureCode, { status: 'PLANNED' }, featuresDir);
+      if (!isBugMode) {
+        const _bp = await getBuildProvider(cwd);
+        const _feat = await _bp.getFeature(featureCode);
+        if (_feat) {
+          // Raw write back to PLANNED — no transition policy, no events, no renderRoadmap.
+          // Matches original updateFeature semantics; keeps teardown side-effect-free.
+          await _bp.persistFeatureRaw(featureCode, { ..._feat, status: 'PLANNED' });
+        }
+      }
       const termState = readActiveBuild(dataDir);
       if (termState) {
         writeActiveBuild(dataDir, { ...termState, status: 'failed', completedAt: new Date().toISOString() });

package/lib/changelog-writer.js

@@ -20,14 +20,19 @@
  * CLI, or future REST routes.
  */
 
-import {
-  readFileSync, writeFileSync, existsSync, mkdirSync, unlinkSync, renameSync,
-} from 'node:fs';
-import { join, dirname } from 'node:path';
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
 
-import { appendEvent, normalizeSince } from './feature-events.js';
+import { normalizeSince } from './feature-events.js';
 import { checkOrInsert } from './idempotency.js';
 
+// providerFor is imported lazily to avoid the load-time cycle:
+// factory.js → local-provider.js → changelog-writer.js
+async function getProvider(cwd) {
+  const { providerFor } = await import('./tracker/factory.js');
+  return providerFor(cwd);
+}
+
 // ---------------------------------------------------------------------------
 // Validation helpers
 // ---------------------------------------------------------------------------
@@ -304,9 +309,13 @@ function readChangelogText(cwd) {
   return readFileSync(p, 'utf-8');
 }
 
-function safeAppendEvent(cwd, event) {
+// Routes through provider.appendEvent so GitHubProvider can post
+// <!--compose-event--> comments. LocalFileProvider delegates to
+// feature-events.js#appendEvent producing byte-identical output.
+async function safeAppendEvent(cwd, event) {
   try {
-    appendEvent(cwd, event);
+    const provider = await getProvider(cwd);
+    await provider.appendEvent(event.code, event);
   } catch (err) {
     // eslint-disable-next-line no-console
     console.warn(`[changelog-writer] audit append failed for ${event.tool} ${event.code ?? ''}: ${err.message}`);
@@ -320,17 +329,86 @@ function maybeIdempotent(args, fn) {
   return Promise.resolve().then(fn);
 }
 
-function atomicWrite(cwd, content) {
-  const p = changelogPath(cwd);
-  const tmp = p + '.tmp';
-  mkdirSync(dirname(p), { recursive: true });
-  try {
-    writeFileSync(tmp, content);
-    renameSync(tmp, p);
-  } catch (err) {
-    try { if (existsSync(tmp)) unlinkSync(tmp); } catch { /* ignore */ }
-    throw err;
+/**
+ * spliceChangelog(currentText, entry) — pure string-in / string-out transform.
+ *
+ * Parses `currentText`, applies the mutation described by `entry` (same shape
+ * as the `args` to addChangelogEntry), and returns the new file content.
+ *
+ * Exported so future providers (e.g. GitHubProvider) can fetch a remote blob,
+ * pass it here, then PUT the result back — without touching the local FS.
+ *
+ * Throws formatError if the file exists but lacks the `# Changelog` header.
+ * Returns { content, insertedAtLine, surface, action } — callers write content.
+ */
+export function spliceChangelog(currentText, entry) {
+  const text = currentText;
+  if (text.length > 0 && !H1_RE.test(text.split('\n')[0] ?? '')) {
+    throw formatError('first line must be "# Changelog" (line 1)');
+  }
+
+  const parsed = parseChangelog(text);
+
+  // Find existing entry across all matching surfaces.
+  const matchingSurfaces = parsed.surfaces.filter(s => s.label === entry.date_or_version);
+  let existingSurface = null;
+  let existingEntry = null;
+  for (const s of matchingSurfaces) {
+    const e = s.entries.find(en => en.code === entry.code);
+    if (e) { existingSurface = s; existingEntry = e; break; }
+  }
+
+  const rendered = renderEntry({
+    code: entry.code,
+    summary: entry.summary,
+    body: entry.body,
+    sections: entry.sections,
+  });
+
+  let action;
+  let chosenSurfaceLabel = entry.date_or_version;
+  let chosenSurfaceStartLine = -1;
+
+  if (existingEntry && !entry.force) {
+    // Storage-level idempotent no-op — caller decides what to do with this.
+    return {
+      content: null,
+      insertedAtLine: existingEntry.startLine,
+      surface: existingSurface.label,
+      idempotent: true,
+    };
   }
+
+  if (existingEntry && entry.force) {
+    action = { kind: 'replace', entry: existingEntry, code: entry.code };
+    chosenSurfaceLabel = existingSurface.label;
+    chosenSurfaceStartLine = existingSurface.startLine;
+  } else if (matchingSurfaces.length > 0) {
+    const surface = matchingSurfaces[0];
+    action = { kind: 'append-to-surface', surface, code: entry.code };
+    chosenSurfaceLabel = surface.label;
+    chosenSurfaceStartLine = surface.startLine;
+  } else if (text.length === 0) {
+    action = { kind: 'new-file', code: entry.code };
+    chosenSurfaceStartLine = 3;
+  } else {
+    action = { kind: 'new-surface', code: entry.code };
+  }
+
+  const { content, insertedAtLine } = buildNextContent(text, parsed, action, rendered, entry.date_or_version);
+
+  // For new-surface, recompute chosenSurfaceStartLine from output.
+  if (action.kind === 'new-surface' || action.kind === 'new-file') {
+    const outLines = content.split('\n');
+    for (let i = 0; i < outLines.length; i++) {
+      if (outLines[i] === `## ${entry.date_or_version}`) {
+        chosenSurfaceStartLine = i + 1;
+        break;
+      }
+    }
+  }
+
+  return { content, insertedAtLine, surface: chosenSurfaceLabel, idempotent: false, action, chosenSurfaceStartLine };
 }
 
 /**
@@ -507,91 +585,41 @@ export async function addChangelogEntry(cwd, args) {
   validateSummary(args.summary);
   validateSections(args.sections);
 
-  return maybeIdempotent({ ...args, cwd }, () => {
-    const text = readChangelogText(cwd);
-    if (text.length > 0 && !H1_RE.test(text.split('\n')[0] ?? '')) {
-      throw formatError('first line must be "# Changelog" (line 1)');
-    }
+  return maybeIdempotent({ ...args, cwd }, async () => {
+    // Route through provider low-level primitives (getChangelog / putChangelog).
+    // Do NOT call provider.appendChangelog — that's LocalFileProvider→addChangelogEntry = recursion.
+    const provider = await getProvider(cwd);
+    const text = await provider.getChangelog();
 
-    const parsed = parseChangelog(text);
-
-    // Find existing entry across all matching surfaces.
-    const matchingSurfaces = parsed.surfaces.filter(s => s.label === args.date_or_version);
-    let existingSurface = null;
-    let existingEntry = null;
-    for (const s of matchingSurfaces) {
-      const e = s.entries.find(en => en.code === args.code);
-      if (e) { existingSurface = s; existingEntry = e; break; }
-    }
+    const spliced = spliceChangelog(text, args);
 
-    const rendered = renderEntry({
-      code: args.code,
-      summary: args.summary,
-      body: args.body,
-      sections: args.sections,
-    });
-
-    let action;
-    let chosenSurfaceLabel = args.date_or_version;
-    let chosenSurfaceStartLine = -1;
-
-    if (existingEntry && !args.force) {
+    if (spliced.idempotent) {
       // Storage-level idempotent no-op: no file write, no audit event
       // (Decision 2 of design). Caller-supplied idempotency_key replays are
       // handled separately by the maybeIdempotent wrapper above.
       return {
-        inserted_at: existingEntry.startLine,
+        inserted_at: spliced.insertedAtLine,
         idempotent: true,
-        surface: existingSurface.label,
+        surface: spliced.surface,
       };
     }
 
-    if (existingEntry && args.force) {
-      action = { kind: 'replace', entry: existingEntry, code: args.code };
-      chosenSurfaceLabel = existingSurface.label;
-      chosenSurfaceStartLine = existingSurface.startLine;
-    } else if (matchingSurfaces.length > 0) {
-      // No existing entry; insert into first (topmost) matching surface.
-      const surface = matchingSurfaces[0];
-      action = { kind: 'append-to-surface', surface, code: args.code };
-      chosenSurfaceLabel = surface.label;
-      chosenSurfaceStartLine = surface.startLine;
-    } else if (text.length === 0) {
-      action = { kind: 'new-file', code: args.code };
-      chosenSurfaceStartLine = 3;  // line of new surface heading after H1 + blank
-    } else {
-      action = { kind: 'new-surface', code: args.code };
-    }
-
-    const { content, insertedAtLine } = buildNextContent(text, parsed, action, rendered, args.date_or_version);
-
-    // For new-surface, recompute chosenSurfaceStartLine from output.
-    if (action.kind === 'new-surface' || action.kind === 'new-file') {
-      const outLines = content.split('\n');
-      for (let i = 0; i < outLines.length; i++) {
-        if (outLines[i] === `## ${args.date_or_version}`) {
-          chosenSurfaceStartLine = i + 1;
-          break;
-        }
-      }
-    }
-
-    atomicWrite(cwd, content);
+    await provider.putChangelog(spliced.content);
 
     const event = {
       tool: 'add_changelog_entry',
       code: args.code,
-      surface_label: chosenSurfaceLabel,
-      surface_start_line: chosenSurfaceStartLine,
+      surface_label: spliced.surface,
+      surface_start_line: spliced.chosenSurfaceStartLine,
     };
     if (args.idempotency_key) event.idempotency_key = args.idempotency_key;
-    if (action.kind === 'replace') event.force = true;
-    safeAppendEvent(cwd, event);
+    if (spliced.action?.kind === 'replace') event.force = true;
+    await safeAppendEvent(cwd, event);
 
     return {
-      inserted_at: insertedAtLine,
+      inserted_at: spliced.insertedAtLine,
       idempotent: false,
-      surface: chosenSurfaceLabel,
+      surface: spliced.surface,
     };
   });
 }

package/lib/completion-writer.js

@@ -24,12 +24,19 @@
 import { mkdirSync, rmSync, statSync } from 'fs';
 import { join, dirname, posix } from 'path';
 
-import { readFeature, updateFeature, listFeatures } from './feature-json.js';
+import { readFeature, listFeatures } from './feature-json.js';
 import { loadFeaturesDir } from './project-paths.js';
-import { appendEvent, normalizeSince } from './feature-events.js';
+import { normalizeSince } from './feature-events.js';
 import { checkOrInsert } from './idempotency.js';
 import { FEATURE_CODE_RE_STRICT as FEATURE_CODE_RE } from './feature-code.js';
 
+// providerFor is imported lazily to avoid the load-time cycle:
+// factory.js → local-provider.js → completion-writer.js
+async function getProvider(cwd) {
+  const { providerFor } = await import('./tracker/factory.js');
+  return providerFor(cwd);
+}
+
 // ---------------------------------------------------------------------------
 // Constants + regexes
 // ---------------------------------------------------------------------------
@@ -206,11 +213,17 @@ function maybeIdempotent(args, fn) {
 // ---------------------------------------------------------------------------
 // safeAppendEvent — best-effort; failed append must NOT roll back a committed
 // mutation (per sibling-writer convention).
+//
+// Routes through provider.appendEvent so GitHubProvider can post
+// <!--compose-event--> comments. LocalFileProvider delegates to
+// feature-events.js#appendEvent producing byte-identical output.
 // ---------------------------------------------------------------------------
 
-function safeAppendEvent(cwd, event) {
+async function safeAppendEvent(cwd, event) {
   try {
-    appendEvent(cwd, event);
+    const { providerFor } = await import('./tracker/factory.js');
+    const provider = await providerFor(cwd);
+    await provider.appendEvent(event.code, event);
   } catch (err) {
     // eslint-disable-next-line no-console
     console.warn(
@@ -252,14 +265,16 @@ export async function recordCompletion(cwd, args) {
   // 3. Compute completion_id
   const completion_id = `${feature_code}:${commit_sha}`;
 
-  const featuresDir = loadFeaturesDir(cwd);
   // 4. Wrap in maybeIdempotent for caller-key path
   return maybeIdempotent({ ...args, cwd }, async () => {
+    // Acquire provider once per operation (resolves factory, no per-call overhead)
+    const provider = await getProvider(cwd);
+
     // 5a. Acquire per-feature advisory lock (Decision 10)
     const release = await acquireFeatureLock(cwd, feature_code);
     try {
-      // 5b. Read feature
-      const feature = readFeature(cwd, feature_code, featuresDir);
+      // 5b. Read feature via provider (low-level primitive — no recursion risk)
+      const feature = await provider.getFeature(feature_code);
       if (!feature) throw notFoundError(feature_code);
 
       // 5c. Snapshot completions array
@@ -303,7 +318,9 @@ export async function recordCompletion(cwd, args) {
       }
 
       // 5h. Persist completion record BEFORE status flip (so flip failure doesn't lose the record)
-      updateFeature(cwd, feature_code, { completions }, featuresDir);
+      // Use persistFeatureRaw (policy-free, no status-delta guard) — completion-writer
+      // owns its own ordering/locking and only mutates the completions array, not status.
+      await provider.persistFeatureRaw(feature_code, { ...feature, completions });
 
       // 5i. Status flip (default on)
       const set_status = args.set_status !== false;
@@ -357,7 +374,7 @@ export async function recordCompletion(cwd, args) {
       };
       if (args.force) auditEvent.force = args.force;
       if (args.idempotency_key) auditEvent.idempotency_key = args.idempotency_key;
-      safeAppendEvent(cwd, auditEvent);
+      await safeAppendEvent(cwd, auditEvent);
 
       // 5l. Return
       return {