aw-ecc 1.4.21

package/scripts/lib/skill-evolution/provenance.js

@@ -0,0 +1,187 @@
+'use strict';
+
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+
+const { ensureDir } = require('../utils');
+
+const PROVENANCE_FILE_NAME = '.provenance.json';
+const SKILL_TYPES = Object.freeze({
+  CURATED: 'curated',
+  LEARNED: 'learned',
+  IMPORTED: 'imported',
+  UNKNOWN: 'unknown',
+});
+
+function resolveRepoRoot(repoRoot) {
+  if (repoRoot) {
+    return path.resolve(repoRoot);
+  }
+
+  return path.resolve(__dirname, '..', '..', '..');
+}
+
+function resolveHomeDir(homeDir) {
+  return homeDir ? path.resolve(homeDir) : os.homedir();
+}
+
+function normalizeSkillDir(skillPath) {
+  if (!skillPath || typeof skillPath !== 'string') {
+    throw new Error('skillPath is required');
+  }
+
+  const resolvedPath = path.resolve(skillPath);
+  if (path.basename(resolvedPath) === 'SKILL.md') {
+    return path.dirname(resolvedPath);
+  }
+
+  return resolvedPath;
+}
+
+function isWithinRoot(targetPath, rootPath) {
+  const relativePath = path.relative(rootPath, targetPath);
+  return relativePath === '' || (
+    !relativePath.startsWith('..')
+    && !path.isAbsolute(relativePath)
+  );
+}
+
+function getSkillRoots(options = {}) {
+  const repoRoot = resolveRepoRoot(options.repoRoot);
+  const homeDir = resolveHomeDir(options.homeDir);
+
+  return {
+    curated: path.join(repoRoot, 'skills'),
+    learned: path.join(homeDir, '.claude', 'skills', 'learned'),
+    imported: path.join(homeDir, '.claude', 'skills', 'imported'),
+  };
+}
+
+function classifySkillPath(skillPath, options = {}) {
+  const skillDir = normalizeSkillDir(skillPath);
+  const roots = getSkillRoots(options);
+
+  if (isWithinRoot(skillDir, roots.curated)) {
+    return SKILL_TYPES.CURATED;
+  }
+
+  if (isWithinRoot(skillDir, roots.learned)) {
+    return SKILL_TYPES.LEARNED;
+  }
+
+  if (isWithinRoot(skillDir, roots.imported)) {
+    return SKILL_TYPES.IMPORTED;
+  }
+
+  return SKILL_TYPES.UNKNOWN;
+}
+
+function requiresProvenance(skillPath, options = {}) {
+  const skillType = classifySkillPath(skillPath, options);
+  return skillType === SKILL_TYPES.LEARNED || skillType === SKILL_TYPES.IMPORTED;
+}
+
+function getProvenancePath(skillPath) {
+  return path.join(normalizeSkillDir(skillPath), PROVENANCE_FILE_NAME);
+}
+
+function isIsoTimestamp(value) {
+  if (typeof value !== 'string' || value.trim().length === 0) {
+    return false;
+  }
+
+  const timestamp = Date.parse(value);
+  return !Number.isNaN(timestamp);
+}
+
+function validateProvenance(record) {
+  const errors = [];
+
+  if (!record || typeof record !== 'object' || Array.isArray(record)) {
+    errors.push('provenance record must be an object');
+    return {
+      valid: false,
+      errors,
+    };
+  }
+
+  if (typeof record.source !== 'string' || record.source.trim().length === 0) {
+    errors.push('source is required');
+  }
+
+  if (!isIsoTimestamp(record.created_at)) {
+    errors.push('created_at must be an ISO timestamp');
+  }
+
+  if (typeof record.confidence !== 'number' || Number.isNaN(record.confidence)) {
+    errors.push('confidence must be a number');
+  } else if (record.confidence < 0 || record.confidence > 1) {
+    errors.push('confidence must be between 0 and 1');
+  }
+
+  if (typeof record.author !== 'string' || record.author.trim().length === 0) {
+    errors.push('author is required');
+  }
+
+  return {
+    valid: errors.length === 0,
+    errors,
+  };
+}
+
+function assertValidProvenance(record) {
+  const validation = validateProvenance(record);
+  if (!validation.valid) {
+    throw new Error(`Invalid provenance metadata: ${validation.errors.join('; ')}`);
+  }
+}
+
+function readProvenance(skillPath, options = {}) {
+  const skillDir = normalizeSkillDir(skillPath);
+  const provenancePath = getProvenancePath(skillDir);
+  const provenanceRequired = options.required === true || requiresProvenance(skillDir, options);
+
+  if (!fs.existsSync(provenancePath)) {
+    if (provenanceRequired) {
+      throw new Error(`Missing provenance metadata for ${skillDir}`);
+    }
+
+    return null;
+  }
+
+  const record = JSON.parse(fs.readFileSync(provenancePath, 'utf8'));
+  assertValidProvenance(record);
+  return record;
+}
+
+function writeProvenance(skillPath, record, options = {}) {
+  const skillDir = normalizeSkillDir(skillPath);
+
+  if (!requiresProvenance(skillDir, options)) {
+    throw new Error(`Provenance metadata is only required for learned or imported skills: ${skillDir}`);
+  }
+
+  assertValidProvenance(record);
+
+  const provenancePath = getProvenancePath(skillDir);
+  ensureDir(skillDir);
+  fs.writeFileSync(provenancePath, `${JSON.stringify(record, null, 2)}\n`, 'utf8');
+
+  return {
+    path: provenancePath,
+    record: { ...record },
+  };
+}
+
+module.exports = {
+  PROVENANCE_FILE_NAME,
+  SKILL_TYPES,
+  classifySkillPath,
+  getProvenancePath,
+  getSkillRoots,
+  readProvenance,
+  requiresProvenance,
+  validateProvenance,
+  writeProvenance,
+};

package/scripts/lib/skill-evolution/tracker.js

@@ -0,0 +1,146 @@
+'use strict';
+
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+
+const { appendFile } = require('../utils');
+
+const VALID_OUTCOMES = new Set(['success', 'failure', 'partial']);
+const VALID_FEEDBACK = new Set(['accepted', 'corrected', 'rejected']);
+
+function resolveHomeDir(homeDir) {
+  return homeDir ? path.resolve(homeDir) : os.homedir();
+}
+
+function getRunsFilePath(options = {}) {
+  if (options.runsFilePath) {
+    return path.resolve(options.runsFilePath);
+  }
+
+  return path.join(resolveHomeDir(options.homeDir), '.claude', 'state', 'skill-runs.jsonl');
+}
+
+function toNullableNumber(value, fieldName) {
+  if (value === null || typeof value === 'undefined') {
+    return null;
+  }
+
+  const numericValue = Number(value);
+  if (!Number.isFinite(numericValue)) {
+    throw new Error(`${fieldName} must be a number`);
+  }
+
+  return numericValue;
+}
+
+function normalizeExecutionRecord(input, options = {}) {
+  if (!input || typeof input !== 'object' || Array.isArray(input)) {
+    throw new Error('skill execution payload must be an object');
+  }
+
+  const skillId = input.skill_id || input.skillId;
+  const skillVersion = input.skill_version || input.skillVersion;
+  const taskDescription = input.task_description || input.task_attempted || input.taskAttempted;
+  const outcome = input.outcome;
+  const recordedAt = input.recorded_at || options.now || new Date().toISOString();
+  const userFeedback = input.user_feedback || input.userFeedback || null;
+
+  if (typeof skillId !== 'string' || skillId.trim().length === 0) {
+    throw new Error('skill_id is required');
+  }
+
+  if (typeof skillVersion !== 'string' || skillVersion.trim().length === 0) {
+    throw new Error('skill_version is required');
+  }
+
+  if (typeof taskDescription !== 'string' || taskDescription.trim().length === 0) {
+    throw new Error('task_description is required');
+  }
+
+  if (!VALID_OUTCOMES.has(outcome)) {
+    throw new Error('outcome must be one of success, failure, or partial');
+  }
+
+  if (userFeedback !== null && !VALID_FEEDBACK.has(userFeedback)) {
+    throw new Error('user_feedback must be accepted, corrected, rejected, or null');
+  }
+
+  if (Number.isNaN(Date.parse(recordedAt))) {
+    throw new Error('recorded_at must be an ISO timestamp');
+  }
+
+  return {
+    skill_id: skillId,
+    skill_version: skillVersion,
+    task_description: taskDescription,
+    outcome,
+    failure_reason: input.failure_reason || input.failureReason || null,
+    tokens_used: toNullableNumber(input.tokens_used ?? input.tokensUsed, 'tokens_used'),
+    duration_ms: toNullableNumber(input.duration_ms ?? input.durationMs, 'duration_ms'),
+    user_feedback: userFeedback,
+    recorded_at: recordedAt,
+  };
+}
+
+function readJsonl(filePath) {
+  if (!fs.existsSync(filePath)) {
+    return [];
+  }
+
+  return fs.readFileSync(filePath, 'utf8')
+    .split('\n')
+    .map(line => line.trim())
+    .filter(Boolean)
+    .reduce((rows, line) => {
+      try {
+        rows.push(JSON.parse(line));
+      } catch {
+        // Ignore malformed rows so analytics remain best-effort.
+      }
+      return rows;
+    }, []);
+}
+
+function recordSkillExecution(input, options = {}) {
+  const record = normalizeExecutionRecord(input, options);
+
+  if (options.stateStore && typeof options.stateStore.recordSkillExecution === 'function') {
+    try {
+      const result = options.stateStore.recordSkillExecution(record);
+      return {
+        storage: 'state-store',
+        record,
+        result,
+      };
+    } catch {
+      // Fall back to JSONL until the formal state-store exists on this branch.
+    }
+  }
+
+  const runsFilePath = getRunsFilePath(options);
+  appendFile(runsFilePath, `${JSON.stringify(record)}\n`);
+
+  return {
+    storage: 'jsonl',
+    path: runsFilePath,
+    record,
+  };
+}
+
+function readSkillExecutionRecords(options = {}) {
+  if (options.stateStore && typeof options.stateStore.listSkillExecutionRecords === 'function') {
+    return options.stateStore.listSkillExecutionRecords();
+  }
+
+  return readJsonl(getRunsFilePath(options));
+}
+
+module.exports = {
+  VALID_FEEDBACK,
+  VALID_OUTCOMES,
+  getRunsFilePath,
+  normalizeExecutionRecord,
+  readSkillExecutionRecords,
+  recordSkillExecution,
+};

package/scripts/lib/skill-evolution/versioning.js

@@ -0,0 +1,237 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const { appendFile, ensureDir } = require('../utils');
+
+const VERSION_DIRECTORY_NAME = '.versions';
+const EVOLUTION_DIRECTORY_NAME = '.evolution';
+const EVOLUTION_LOG_TYPES = Object.freeze([
+  'observations',
+  'inspections',
+  'amendments',
+]);
+
+function normalizeSkillDir(skillPath) {
+  if (!skillPath || typeof skillPath !== 'string') {
+    throw new Error('skillPath is required');
+  }
+
+  const resolvedPath = path.resolve(skillPath);
+  if (path.basename(resolvedPath) === 'SKILL.md') {
+    return path.dirname(resolvedPath);
+  }
+
+  return resolvedPath;
+}
+
+function getSkillFilePath(skillPath) {
+  return path.join(normalizeSkillDir(skillPath), 'SKILL.md');
+}
+
+function ensureSkillExists(skillPath) {
+  const skillFilePath = getSkillFilePath(skillPath);
+  if (!fs.existsSync(skillFilePath)) {
+    throw new Error(`Skill file not found: ${skillFilePath}`);
+  }
+
+  return skillFilePath;
+}
+
+function getVersionsDir(skillPath) {
+  return path.join(normalizeSkillDir(skillPath), VERSION_DIRECTORY_NAME);
+}
+
+function getEvolutionDir(skillPath) {
+  return path.join(normalizeSkillDir(skillPath), EVOLUTION_DIRECTORY_NAME);
+}
+
+function getEvolutionLogPath(skillPath, logType) {
+  if (!EVOLUTION_LOG_TYPES.includes(logType)) {
+    throw new Error(`Unknown evolution log type: ${logType}`);
+  }
+
+  return path.join(getEvolutionDir(skillPath), `${logType}.jsonl`);
+}
+
+function ensureSkillVersioning(skillPath) {
+  ensureSkillExists(skillPath);
+
+  const versionsDir = getVersionsDir(skillPath);
+  const evolutionDir = getEvolutionDir(skillPath);
+
+  ensureDir(versionsDir);
+  ensureDir(evolutionDir);
+
+  for (const logType of EVOLUTION_LOG_TYPES) {
+    const logPath = getEvolutionLogPath(skillPath, logType);
+    if (!fs.existsSync(logPath)) {
+      fs.writeFileSync(logPath, '', 'utf8');
+    }
+  }
+
+  return {
+    versionsDir,
+    evolutionDir,
+  };
+}
+
+function parseVersionNumber(fileName) {
+  const match = /^v(\d+)\.md$/.exec(fileName);
+  if (!match) {
+    return null;
+  }
+
+  return Number(match[1]);
+}
+
+function listVersions(skillPath) {
+  const versionsDir = getVersionsDir(skillPath);
+  if (!fs.existsSync(versionsDir)) {
+    return [];
+  }
+
+  return fs.readdirSync(versionsDir)
+    .map(fileName => {
+      const version = parseVersionNumber(fileName);
+      if (version === null) {
+        return null;
+      }
+
+      const filePath = path.join(versionsDir, fileName);
+      const stats = fs.statSync(filePath);
+
+      return {
+        version,
+        path: filePath,
+        created_at: stats.mtime.toISOString(),
+      };
+    })
+    .filter(Boolean)
+    .sort((left, right) => left.version - right.version);
+}
+
+function getCurrentVersion(skillPath) {
+  const skillFilePath = getSkillFilePath(skillPath);
+  if (!fs.existsSync(skillFilePath)) {
+    return 0;
+  }
+
+  const versions = listVersions(skillPath);
+  if (versions.length === 0) {
+    return 1;
+  }
+
+  return versions[versions.length - 1].version;
+}
+
+function appendEvolutionRecord(skillPath, logType, record) {
+  ensureSkillVersioning(skillPath);
+  appendFile(getEvolutionLogPath(skillPath, logType), `${JSON.stringify(record)}\n`);
+  return { ...record };
+}
+
+function readJsonl(filePath) {
+  if (!fs.existsSync(filePath)) {
+    return [];
+  }
+
+  return fs.readFileSync(filePath, 'utf8')
+    .split('\n')
+    .map(line => line.trim())
+    .filter(Boolean)
+    .reduce((rows, line) => {
+      try {
+        rows.push(JSON.parse(line));
+      } catch {
+        // Ignore malformed rows so the log remains append-only and resilient.
+      }
+      return rows;
+    }, []);
+}
+
+function getEvolutionLog(skillPath, logType) {
+  return readJsonl(getEvolutionLogPath(skillPath, logType));
+}
+
+function createVersion(skillPath, options = {}) {
+  const skillFilePath = ensureSkillExists(skillPath);
+  ensureSkillVersioning(skillPath);
+
+  const versions = listVersions(skillPath);
+  const nextVersion = versions.length === 0 ? 1 : versions[versions.length - 1].version + 1;
+  const snapshotPath = path.join(getVersionsDir(skillPath), `v${nextVersion}.md`);
+  const skillContent = fs.readFileSync(skillFilePath, 'utf8');
+  const createdAt = options.timestamp || new Date().toISOString();
+
+  fs.writeFileSync(snapshotPath, skillContent, 'utf8');
+  appendEvolutionRecord(skillPath, 'amendments', {
+    event: 'snapshot',
+    version: nextVersion,
+    reason: options.reason || null,
+    author: options.author || null,
+    status: 'applied',
+    created_at: createdAt,
+  });
+
+  return {
+    version: nextVersion,
+    path: snapshotPath,
+    created_at: createdAt,
+  };
+}
+
+function rollbackTo(skillPath, targetVersion, options = {}) {
+  const normalizedTargetVersion = Number(targetVersion);
+  if (!Number.isInteger(normalizedTargetVersion) || normalizedTargetVersion <= 0) {
+    throw new Error(`Invalid target version: ${targetVersion}`);
+  }
+
+  ensureSkillExists(skillPath);
+  ensureSkillVersioning(skillPath);
+
+  const targetPath = path.join(getVersionsDir(skillPath), `v${normalizedTargetVersion}.md`);
+  if (!fs.existsSync(targetPath)) {
+    throw new Error(`Version not found: v${normalizedTargetVersion}`);
+  }
+
+  const currentVersion = getCurrentVersion(skillPath);
+  const targetContent = fs.readFileSync(targetPath, 'utf8');
+  fs.writeFileSync(getSkillFilePath(skillPath), targetContent, 'utf8');
+
+  const createdVersion = createVersion(skillPath, {
+    timestamp: options.timestamp,
+    reason: options.reason || `rollback to v${normalizedTargetVersion}`,
+    author: options.author || null,
+  });
+
+  appendEvolutionRecord(skillPath, 'amendments', {
+    event: 'rollback',
+    version: createdVersion.version,
+    source_version: currentVersion,
+    target_version: normalizedTargetVersion,
+    reason: options.reason || null,
+    author: options.author || null,
+    status: 'applied',
+    created_at: options.timestamp || new Date().toISOString(),
+  });
+
+  return createdVersion;
+}
+
+module.exports = {
+  EVOLUTION_DIRECTORY_NAME,
+  EVOLUTION_LOG_TYPES,
+  VERSION_DIRECTORY_NAME,
+  appendEvolutionRecord,
+  createVersion,
+  ensureSkillVersioning,
+  getCurrentVersion,
+  getEvolutionDir,
+  getEvolutionLog,
+  getEvolutionLogPath,
+  getVersionsDir,
+  listVersions,
+  rollbackTo,
+};

package/scripts/lib/skill-improvement/amendify.js

@@ -0,0 +1,89 @@
+'use strict';
+
+const { buildSkillHealthReport } = require('./health');
+
+const AMENDMENT_SCHEMA_VERSION = 'ecc.skill-amendment-proposal.v1';
+
+function createProposalId(skillId) {
+  return `amend-${skillId}-${Date.now()}`;
+}
+
+function summarizePatchPreview(skillId, health) {
+  const lines = [
+    '## Failure-Driven Amendments',
+    '',
+    `- Focus skill routing for \`${skillId}\` when tasks match the proven success cases.`,
+  ];
+
+  if (health.recurringErrors[0]) {
+    lines.push(`- Add explicit guardrails for recurring failure: ${health.recurringErrors[0].error}.`);
+  }
+
+  if (health.recurringTasks[0]) {
+    lines.push(`- Add an example workflow for task pattern: ${health.recurringTasks[0].task}.`);
+  }
+
+  if (health.recurringFeedback[0]) {
+    lines.push(`- Address repeated user feedback: ${health.recurringFeedback[0].feedback}.`);
+  }
+
+  lines.push('- Add a verification checklist before declaring the skill output complete.');
+  return lines.join('\n');
+}
+
+function proposeSkillAmendment(skillId, records, options = {}) {
+  const report = buildSkillHealthReport(records, {
+    ...options,
+    skillId,
+    minFailureCount: options.minFailureCount || 1
+  });
+  const [health] = report.skills;
+
+  if (!health || health.failures === 0) {
+    return {
+      schemaVersion: AMENDMENT_SCHEMA_VERSION,
+      skill: {
+        id: skillId,
+        path: null
+      },
+      status: 'insufficient-evidence',
+      rationale: ['No failed observations were available for this skill.'],
+      patch: null
+    };
+  }
+
+  const preview = summarizePatchPreview(skillId, health);
+
+  return {
+    schemaVersion: AMENDMENT_SCHEMA_VERSION,
+    proposalId: createProposalId(skillId),
+    generatedAt: new Date().toISOString(),
+    status: 'proposed',
+    skill: {
+      id: skillId,
+      path: health.skill.path || null
+    },
+    evidence: {
+      totalRuns: health.totalRuns,
+      failures: health.failures,
+      successRate: health.successRate,
+      recurringErrors: health.recurringErrors,
+      recurringTasks: health.recurringTasks,
+      recurringFeedback: health.recurringFeedback
+    },
+    rationale: [
+      'Proposals are generated from repeated failed runs rather than a single anecdotal error.',
+      'The suggested patch is additive so the original SKILL.md intent remains auditable.'
+    ],
+    patch: {
+      format: 'markdown-fragment',
+      targetPath: health.skill.path || `skills/${skillId}/SKILL.md`,
+      preview
+    }
+  };
+}
+
+module.exports = {
+  AMENDMENT_SCHEMA_VERSION,
+  proposeSkillAmendment
+};

package/scripts/lib/skill-improvement/evaluate.js

@@ -0,0 +1,59 @@
+'use strict';
+
+const EVALUATION_SCHEMA_VERSION = 'ecc.skill-evaluation.v1';
+
+function roundRate(value) {
+  return Math.round(value * 1000) / 1000;
+}
+
+function summarize(records) {
+  const runs = records.length;
+  const successes = records.filter(record => record.outcome && record.outcome.success).length;
+  const failures = runs - successes;
+  return {
+    runs,
+    successes,
+    failures,
+    successRate: runs > 0 ? roundRate(successes / runs) : 0
+  };
+}
+
+function buildSkillEvaluationScaffold(skillId, records, options = {}) {
+  const minimumRunsPerVariant = options.minimumRunsPerVariant || 2;
+  const amendmentId = options.amendmentId || null;
+  const filtered = records.filter(record => record.skill && record.skill.id === skillId);
+  const baseline = filtered.filter(record => !record.run || record.run.variant !== 'amended');
+  const amended = filtered.filter(record => record.run && record.run.variant === 'amended')
+    .filter(record => !amendmentId || record.run.amendmentId === amendmentId);
+
+  const baselineSummary = summarize(baseline);
+  const amendedSummary = summarize(amended);
+  const delta = {
+    successRate: roundRate(amendedSummary.successRate - baselineSummary.successRate),
+    failures: amendedSummary.failures - baselineSummary.failures
+  };
+
+  let recommendation = 'insufficient-data';
+  if (baselineSummary.runs >= minimumRunsPerVariant && amendedSummary.runs >= minimumRunsPerVariant) {
+    recommendation = delta.successRate > 0 ? 'promote-amendment' : 'keep-baseline';
+  }
+
+  return {
+    schemaVersion: EVALUATION_SCHEMA_VERSION,
+    generatedAt: new Date().toISOString(),
+    skillId,
+    amendmentId,
+    gate: {
+      minimumRunsPerVariant
+    },
+    baseline: baselineSummary,
+    amended: amendedSummary,
+    delta,
+    recommendation
+  };
+}
+
+module.exports = {
+  EVALUATION_SCHEMA_VERSION,
+  buildSkillEvaluationScaffold
+};