cognitive-core 0.2.1 → 0.2.2

package/src/memory/curated-loader.ts

@@ -6,17 +6,30 @@
  * drop all curated playbooks and reload from source files without
  * losing extracted or imported playbooks.
  *
+ * Supports recursive directory scanning and source-prefix-based
+ * deletion for per-source recreate (multiple sources don't clobber
+ * each other).
+ *
  * Inspired by Dash's separation of curated knowledge (file-loaded)
  * vs learnings (runtime-discovered).
  */
 
-import { readFile, readdir, stat } from 'node:fs/promises';
-import { join, relative } from 'node:path';
+import { readFile, readdir } from 'node:fs/promises';
+import { join, relative, dirname } from 'node:path';
 import { existsSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
 import type { PlaybookProvenance } from '../types/index.js';
 import { createPlaybook } from '../types/index.js';
 import type { PlaybookLibrary } from './playbook.js';
 
+/**
+ * Path to the bundled compound-engineering seed pack shipped with cognitive-core.
+ * Resolves relative to this file's location in the package.
+ */
+const __filename_curated = fileURLToPath(import.meta.url);
+const __dirname_curated = dirname(__filename_curated);
+export const BUNDLED_SEED_PACK_DIR = join(__dirname_curated, '../../playbooks/compound-engineering');
+
 /**
  * Schema for a curated playbook JSON file
  */
@@ -55,17 +68,49 @@ export interface CuratedLoadResult {
   recreated: boolean;
 }
 
+/**
+ * Options for loadCuratedPlaybooks
+ */
+export interface CuratedLoadOptions {
+  /** If true, delete existing curated playbooks before loading */
+  recreate?: boolean;
+  /**
+   * Source identifier for provenance tracking.
+   * Used as prefix in provenance.sourceFile for per-source recreate.
+   * If provided and recreate is true, only playbooks from this source are deleted.
+   */
+  sourceId?: string;
+}
+
+/**
+ * Recursively find all JSON files in a directory tree.
+ */
+async function findJsonFiles(dir: string): Promise<string[]> {
+  const entries = await readdir(dir, { withFileTypes: true });
+  const files: string[] = [];
+  for (const entry of entries) {
+    const fullPath = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      files.push(...await findJsonFiles(fullPath));
+    } else if (entry.name.endsWith('.json')) {
+      files.push(fullPath);
+    }
+  }
+  return files;
+}
+
 /**
  * Load curated playbooks from a directory of JSON files into the library.
+ * Supports recursive directory scanning.
  *
  * @param curatedDir - Directory containing curated playbook JSON files
  * @param library - PlaybookLibrary to load into
- * @param options.recreate - If true, delete existing curated playbooks before loading
+ * @param options - Load options (recreate, sourceId)
  */
 export async function loadCuratedPlaybooks(
   curatedDir: string,
   library: PlaybookLibrary,
-  options?: { recreate?: boolean }
+  options?: CuratedLoadOptions
 ): Promise<CuratedLoadResult> {
   const result: CuratedLoadResult = {
     loaded: 0,
@@ -78,20 +123,23 @@ export async function loadCuratedPlaybooks(
     return result;
   }
 
-  // Recreate: drop all curated playbooks first
+  // Recreate: drop curated playbooks before loading
   if (options?.recreate) {
-    await library.deleteByOrigin('curated');
+    if (options.sourceId) {
+      // Per-source recreate: only delete playbooks from this source
+      await library.deleteByCuratedSource(options.sourceId);
+    } else {
+      // Global recreate: delete all curated playbooks
+      await library.deleteByOrigin('curated');
+    }
     result.recreated = true;
   }
 
-  // Find all JSON files in the directory (non-recursive)
-  const files = await readdir(curatedDir);
-  const jsonFiles = files.filter((f) => f.endsWith('.json'));
+  // Find all JSON files recursively
+  const jsonFiles = await findJsonFiles(curatedDir);
 
-  for (const file of jsonFiles) {
-    const filePath = join(curatedDir, file);
-    const fileStat = await stat(filePath);
-    if (!fileStat.isFile()) continue;
+  for (const filePath of jsonFiles) {
+    const relativeToDir = relative(curatedDir, filePath);
 
     try {
       const content = await readFile(filePath, 'utf-8');
@@ -100,7 +148,7 @@ export async function loadCuratedPlaybooks(
       // Validate required fields
       if (!parsed.name || !parsed.applicability || !parsed.guidance) {
         result.errors.push({
-          file,
+          file: relativeToDir,
           error: 'Missing required fields: name, applicability, guidance',
         });
         continue;
@@ -113,10 +161,15 @@ export async function loadCuratedPlaybooks(
         continue;
       }
 
+      // Build sourceFile with optional source prefix
       const relativePath = relative(process.cwd(), filePath);
+      const sourceFile = options?.sourceId
+        ? `${options.sourceId}:${relativeToDir}`
+        : relativePath;
+
       const provenance: PlaybookProvenance = {
         origin: 'curated',
-        sourceFile: relativePath,
+        sourceFile,
         curatedBy: parsed.curatedBy,
         recordedAt: new Date(),
       };
@@ -150,7 +203,7 @@ export async function loadCuratedPlaybooks(
       result.loaded++;
     } catch (error) {
       result.errors.push({
-        file,
+        file: relativeToDir,
         error: error instanceof Error ? error.message : String(error),
       });
     }

package/src/memory/index.ts

@@ -99,10 +99,26 @@ export type { SqlitePersistenceConfig } from '../persistence/index.js';
 // Curated playbook loading
 export {
   loadCuratedPlaybooks,
+  BUNDLED_SEED_PACK_DIR,
   type CuratedPlaybookFile,
   type CuratedLoadResult,
+  type CuratedLoadOptions,
 } from './curated-loader.js';
 
+// Source resolver (git + local path resolution for curated sources)
+export {
+  resolvePlaybookSource,
+  isGitSource,
+  parseGitSource,
+  discoverPlaybookDir,
+  sourceStateKey,
+  type ParsedGitSource,
+  type PlaybookManifest,
+  type SourceState,
+  type ResolveResult,
+  type ResolveOptions,
+} from './source-resolver.js';
+
 // Skill export (compile playbooks → Open Agent Skills format)
 export {
   compilePlaybookToSkill,

package/src/memory/playbook.ts

@@ -616,6 +616,25 @@ export class PlaybookLibrary {
     return toDelete.length;
   }
 
+  /**
+   * Delete curated playbooks whose provenance.sourceFile starts with the given prefix.
+   * Used by the source resolver to recreate playbooks from a specific source
+   * without affecting other curated sources.
+   */
+  async deleteByCuratedSource(sourcePrefix: string): Promise<number> {
+    await this.init();
+    const toDelete = this.getAllSync().filter(
+      (p) =>
+        p.provenance?.origin === 'curated' &&
+        p.provenance.sourceFile?.startsWith(sourcePrefix),
+    );
+    for (const playbook of toDelete) {
+      this.persistence.deletePlaybook(playbook.id);
+      this._index.remove(playbook.id);
+    }
+    return toDelete.length;
+  }
+
   /**
    * Get provenance summary across all playbooks
    */

package/src/memory/source-resolver.ts

@@ -0,0 +1,422 @@
+/**
+ * Source Resolver — Normalizes playbook sources (local paths, git URLs, shorthands)
+ * into local directory paths for the curated loader.
+ *
+ * Supports:
+ * - Local filesystem paths (pass-through)
+ * - Git URLs: "git:https://github.com/org/repo" or "git:https://github.com/org/repo#ref"
+ * - GitHub shorthand: "github:org/repo" or "github:org/repo#ref"
+ *
+ * Cloned repos are cached in a configurable directory (default: .atlas/curated-cache/).
+ * Cache freshness is controlled by maxAgeMs. Stale caches are refreshed on next resolve.
+ */
+
+import { execFile as execFileCb } from 'node:child_process';
+import { existsSync, statSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { mkdir, readFile, readdir } from 'node:fs/promises';
+import { promisify } from 'node:util';
+
+const execFile = promisify(execFileCb);
+
+// ============================================================================
+// Types
+// ============================================================================
+
+/**
+ * Parsed git source components
+ */
+export interface ParsedGitSource {
+  url: string;
+  ref?: string;
+  cacheKey: string;
+}
+
+/**
+ * Manifest file (cognitive-core.json) at repo root
+ */
+export interface PlaybookManifest {
+  /** Relative path to playbook directory within the repo */
+  playbooks: string;
+  /** Optional version label */
+  version?: string;
+  /** Optional display name */
+  name?: string;
+  /** Optional description */
+  description?: string;
+}
+
+/**
+ * State persisted per resolved source (stored in system_state)
+ */
+export interface SourceState {
+  /** Original config source string */
+  source: string;
+  /** Resolved git URL (empty for local paths) */
+  url: string;
+  /** Requested ref (branch/tag/commit) */
+  ref: string;
+  /** Actual commit SHA loaded */
+  commitSha: string;
+  /** Local cache path */
+  cachePath: string;
+  /** Discovered playbook directory within cache */
+  playbookDir: string;
+  /** ISO timestamp of last successful load */
+  loadedAt: string;
+  /** Number of playbooks loaded from this source */
+  playbookCount: number;
+}
+
+/**
+ * Result of resolving a source
+ */
+export interface ResolveResult {
+  /** Local directory path containing playbook JSON files */
+  playbookDir: string;
+  /** Whether this is a git source */
+  isGit: boolean;
+  /** Commit SHA (only for git sources) */
+  commitSha?: string;
+  /** Whether the source was freshly cloned/updated */
+  updated: boolean;
+  /** Source identifier for provenance tracking */
+  sourceId: string;
+}
+
+/**
+ * Options for source resolution
+ */
+export interface ResolveOptions {
+  /** Cache directory (default: .atlas/curated-cache) */
+  cacheDir: string;
+  /** Maximum cache age in milliseconds before refreshing (default: 24h) */
+  maxAgeMs: number;
+  /** Force update regardless of cache age */
+  forceUpdate?: boolean;
+  /** Callback for loading/saving source state */
+  getSourceState?: (source: string) => SourceState | undefined;
+  setSourceState?: (source: string, state: SourceState) => void;
+}
+
+// ============================================================================
+// Public API
+// ============================================================================
+
+/**
+ * Check if a source string is a git source (not a local path).
+ */
+export function isGitSource(source: string): boolean {
+  return (
+    source.startsWith('git:') ||
+    source.startsWith('github:') ||
+    source.startsWith('git@') ||
+    source.startsWith('git+')
+  );
+}
+
+/**
+ * Parse a git source string into URL and optional ref components.
+ *
+ * Supported formats:
+ * - "git:https://github.com/org/repo"
+ * - "git:https://github.com/org/repo#v1.0.0"
+ * - "github:org/repo"
+ * - "github:org/repo#branch"
+ */
+export function parseGitSource(source: string): ParsedGitSource {
+  let raw = source;
+
+  // Strip prefix
+  if (raw.startsWith('git:')) {
+    raw = raw.slice(4);
+  } else if (raw.startsWith('git+')) {
+    raw = raw.slice(4);
+  } else if (raw.startsWith('github:')) {
+    const rest = raw.slice(7); // "org/repo" or "org/repo#ref"
+    const [path, ref] = rest.split('#', 2);
+    const url = `https://github.com/${path}.git`;
+    const cacheKey = `github.com/${path}`;
+    return { url, ref, cacheKey };
+  }
+
+  // Parse ref from fragment
+  const hashIdx = raw.indexOf('#');
+  let url: string;
+  let ref: string | undefined;
+
+  if (hashIdx !== -1) {
+    url = raw.slice(0, hashIdx);
+    ref = raw.slice(hashIdx + 1);
+  } else {
+    url = raw;
+  }
+
+  // Ensure .git suffix for HTTPS URLs
+  if (url.startsWith('https://') && !url.endsWith('.git')) {
+    url += '.git';
+  }
+
+  // Build cache key from URL (strip protocol + .git suffix)
+  const cacheKey = url
+    .replace(/^https?:\/\//, '')
+    .replace(/\.git$/, '');
+
+  return { url, ref, cacheKey };
+}
+
+/**
+ * Resolve a playbook source to a local directory path.
+ *
+ * For local paths, returns the path as-is (after verifying it exists).
+ * For git sources, clones/updates to a cache directory and discovers
+ * the playbook directory within the repo.
+ */
+export async function resolvePlaybookSource(
+  source: string,
+  options: ResolveOptions,
+): Promise<ResolveResult | null> {
+  // Local path — pass through
+  if (!isGitSource(source)) {
+    if (!existsSync(source)) {
+      console.warn(`[source-resolver] Local path does not exist: ${source}`);
+      return null;
+    }
+    return {
+      playbookDir: source,
+      isGit: false,
+      updated: false,
+      sourceId: source,
+    };
+  }
+
+  // Git source — parse, clone/update, discover
+  const parsed = parseGitSource(source);
+  const cachePath = join(options.cacheDir, parsed.cacheKey);
+
+  try {
+    // Ensure cache parent dir exists
+    await mkdir(cachePath, { recursive: true });
+
+    const isCloned = existsSync(join(cachePath, '.git'));
+
+    if (!isCloned) {
+      // First-time clone
+      await gitClone(parsed.url, parsed.ref, cachePath);
+    } else if (shouldUpdate(source, options)) {
+      // Update existing cache
+      await gitUpdate(parsed.ref, cachePath);
+    }
+
+    // Get current commit SHA
+    const commitSha = await gitRevParse(cachePath);
+
+    // Check if we already loaded this exact SHA
+    const prevState = options.getSourceState?.(source);
+    if (prevState && prevState.commitSha === commitSha && !options.forceUpdate) {
+      return {
+        playbookDir: prevState.playbookDir,
+        isGit: true,
+        commitSha,
+        updated: false,
+        sourceId: source,
+      };
+    }
+
+    // Discover playbook directory
+    const playbookDir = await discoverPlaybookDir(cachePath);
+    if (!playbookDir) {
+      console.warn(`[source-resolver] No playbook directory found in ${source}`);
+      return null;
+    }
+
+    // Persist source state
+    const state: SourceState = {
+      source,
+      url: parsed.url,
+      ref: parsed.ref ?? '',
+      commitSha,
+      cachePath,
+      playbookDir,
+      loadedAt: new Date().toISOString(),
+      playbookCount: 0, // Updated by caller after loading
+    };
+    options.setSourceState?.(source, state);
+
+    return {
+      playbookDir,
+      isGit: true,
+      commitSha,
+      updated: true,
+      sourceId: source,
+    };
+  } catch (error) {
+    // If we have a cached version, use it despite the error
+    if (existsSync(join(cachePath, '.git'))) {
+      console.warn(
+        `[source-resolver] Failed to update ${source}, using cached version:`,
+        error instanceof Error ? error.message : String(error),
+      );
+      const prevState = options.getSourceState?.(source);
+      if (prevState) {
+        return {
+          playbookDir: prevState.playbookDir,
+          isGit: true,
+          commitSha: prevState.commitSha,
+          updated: false,
+          sourceId: source,
+        };
+      }
+      // Try to discover even without state
+      const playbookDir = await discoverPlaybookDir(cachePath);
+      if (playbookDir) {
+        return {
+          playbookDir,
+          isGit: true,
+          updated: false,
+          sourceId: source,
+        };
+      }
+    }
+
+    console.warn(
+      `[source-resolver] Failed to resolve ${source}, skipping:`,
+      error instanceof Error ? error.message : String(error),
+    );
+    return null;
+  }
+}
+
+// ============================================================================
+// Git operations
+// ============================================================================
+
+async function gitClone(url: string, ref: string | undefined, dest: string): Promise<void> {
+  // Remove the directory first since mkdir already created it
+  rmSync(dest, { recursive: true, force: true });
+
+  const args = ['clone', '--depth', '1', '--single-branch'];
+  if (ref) {
+    args.push('--branch', ref);
+  }
+  args.push(url, dest);
+
+  await execFile('git', args, { timeout: 60_000 });
+}
+
+async function gitUpdate(ref: string | undefined, cachePath: string): Promise<void> {
+  const fetchArgs = ['fetch', 'origin'];
+  if (ref) {
+    fetchArgs.push(ref);
+  }
+  await execFile('git', fetchArgs, { cwd: cachePath, timeout: 30_000 });
+
+  // Always use FETCH_HEAD — works for branches, tags, and commits.
+  // Using origin/${ref} would break for tags (refs/tags/*, not refs/remotes/origin/*).
+  await execFile('git', ['reset', '--hard', 'FETCH_HEAD'], {
+    cwd: cachePath,
+    timeout: 10_000,
+  });
+}
+
+async function gitRevParse(cachePath: string): Promise<string> {
+  const { stdout } = await execFile('git', ['rev-parse', 'HEAD'], {
+    cwd: cachePath,
+    timeout: 5_000,
+  });
+  return stdout.trim();
+}
+
+// ============================================================================
+// Playbook directory discovery
+// ============================================================================
+
+/**
+ * Discover the playbook directory within a cloned repo.
+ *
+ * Search order:
+ * 1. Manifest file (cognitive-core.json) at repo root
+ * 2. Convention: playbooks/ directory
+ * 3. Convention: curated-playbooks/ directory
+ * 4. Root directory if it contains .json files matching schema
+ */
+export async function discoverPlaybookDir(repoRoot: string): Promise<string | null> {
+  // 1. Check for manifest
+  const manifestPath = join(repoRoot, 'cognitive-core.json');
+  if (existsSync(manifestPath)) {
+    try {
+      const content = await readFile(manifestPath, 'utf-8');
+      const manifest = JSON.parse(content) as PlaybookManifest;
+      if (manifest.playbooks) {
+        const dir = join(repoRoot, manifest.playbooks);
+        if (existsSync(dir)) {
+          return dir;
+        }
+      }
+    } catch {
+      // Invalid manifest, fall through to convention
+    }
+  }
+
+  // 2. Convention: playbooks/
+  const playbooksDir = join(repoRoot, 'playbooks');
+  if (existsSync(playbooksDir) && statSync(playbooksDir).isDirectory()) {
+    return playbooksDir;
+  }
+
+  // 3. Convention: curated-playbooks/
+  const curatedDir = join(repoRoot, 'curated-playbooks');
+  if (existsSync(curatedDir) && statSync(curatedDir).isDirectory()) {
+    return curatedDir;
+  }
+
+  // 4. Check if root has JSON files that look like playbooks
+  if (await hasPlaybookJsonFiles(repoRoot)) {
+    return repoRoot;
+  }
+
+  return null;
+}
+
+/**
+ * Check if a directory contains at least one JSON file that looks like a playbook.
+ */
+async function hasPlaybookJsonFiles(dir: string): Promise<boolean> {
+  try {
+    const files = await readdir(dir);
+    const jsonFiles = files.filter(f => f.endsWith('.json'));
+    if (jsonFiles.length === 0) return false;
+
+    // Check first JSON file for required playbook fields
+    const content = await readFile(join(dir, jsonFiles[0]), 'utf-8');
+    const parsed = JSON.parse(content);
+    return !!(parsed.name && parsed.applicability && parsed.guidance);
+  } catch {
+    return false;
+  }
+}
+
+// ============================================================================
+// Cache management
+// ============================================================================
+
+/**
+ * Determine whether a cached source should be updated.
+ */
+function shouldUpdate(source: string, options: ResolveOptions): boolean {
+  if (options.forceUpdate) return true;
+
+  const state = options.getSourceState?.(source);
+  if (!state) return true; // No previous state, always update
+
+  const loadedAt = new Date(state.loadedAt).getTime();
+  const age = Date.now() - loadedAt;
+  return age > options.maxAgeMs;
+}
+
+/**
+ * Generate the system_state key for a source.
+ */
+export function sourceStateKey(source: string): string {
+  return `curated-source:${source}`;
+}

package/src/types/config.ts

@@ -513,6 +513,45 @@ export const SkillTreeConfigSchema = z.object({
   dbPath: z.string().default('skills.db'),
 });
 
+/**
+ * Curated playbook cache configuration (for git sources)
+ */
+export const CuratedCacheConfigSchema = z.object({
+  /** Cache directory for cloned repos (relative to storage.baseDir, default: curated-cache) */
+  dir: z.string().default('curated-cache'),
+  /** Maximum cache age in milliseconds before refreshing (default: 24h) */
+  maxAgeMs: z.number().default(24 * 60 * 60 * 1000),
+});
+
+export type CuratedCacheConfig = z.infer<typeof CuratedCacheConfigSchema>;
+
+/**
+ * Curated playbook configuration
+ */
+export const CuratedPlaybooksConfigSchema = z.object({
+  /** Load the bundled compound-engineering seed pack on init */
+  loadBundledSeedPack: z.boolean().default(false),
+  /** Additional directories to load curated playbooks from (backward compat) */
+  additionalDirs: z.array(z.string()).default([]),
+  /**
+   * Playbook sources — local paths or git URIs.
+   *
+   * Supported formats:
+   * - Local path: "/path/to/playbooks"
+   * - Git URL: "git:https://github.com/org/repo" or "git:https://github.com/org/repo#v1.0.0"
+   * - GitHub shorthand: "github:org/repo" or "github:org/repo#branch"
+   *
+   * Git sources are cloned to a local cache and refreshed based on cache.maxAgeMs.
+   */
+  sources: z.array(z.string()).default([]),
+  /** Cache configuration for git-based sources */
+  cache: CuratedCacheConfigSchema.default({}),
+  /** Re-create curated playbooks on each init (useful when updating seed files) */
+  recreateOnInit: z.boolean().default(false),
+});
+
+export type CuratedPlaybooksConfig = z.infer<typeof CuratedPlaybooksConfigSchema>;
+
 export type SkillTreeConfig = z.infer<typeof SkillTreeConfigSchema>;
 
 /**
@@ -570,6 +609,8 @@ export const AtlasConfigSchema = z.object({
   knowledgeBank: KnowledgeBankConfigSchema.default({}),
   /** Skill-tree storage configuration */
   skillTree: SkillTreeConfigSchema.default({}),
+  /** Curated playbook seeding configuration */
+  curatedPlaybooks: CuratedPlaybooksConfigSchema.default({}),
   /** Session bank configuration (Entire CLI integration) */
   sessionBank: SessionBankConfigSchema.default({}),
   /** Feature toggles for ruvector-inspired subsystems */
@@ -609,6 +650,11 @@ export function mergeConfig(
     teamLearning: { ...base.teamLearning, ...overrides.teamLearning },
     knowledgeBank: { ...base.knowledgeBank, ...overrides.knowledgeBank },
     skillTree: { ...base.skillTree, ...overrides.skillTree },
+    curatedPlaybooks: {
+      ...base.curatedPlaybooks,
+      ...overrides.curatedPlaybooks,
+      cache: { ...base.curatedPlaybooks.cache, ...overrides.curatedPlaybooks?.cache },
+    },
     sessionBank: { ...base.sessionBank, ...overrides.sessionBank },
     features: { ...base.features, ...overrides.features },
   });