@evo-dev/core 0.0.1-alpha.1 → 0.0.1-alpha.11

package/src/evolution/candidates/index.ts

@@ -0,0 +1,564 @@
+import { createHash } from "node:crypto";
+import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { resolveEvoDevPaths } from "../../config/paths.ts";
+import { pathExists, readJsonFiles, writeJsonFile as writeJson } from "../../utils/index.ts";
+import { resolveEvolutionPaths } from "../paths.ts";
+import type {
+  EvolutionEvosCase,
+  EvolutionEvosCaseQueryResult,
+  EvolutionKnowledgeRecord,
+  EvolutionKnowledgeReviewHistoryRecord,
+  EvolutionRepoProposal,
+  EvolutionReviewCandidate,
+  EvolutionReviewSnapshot,
+  EvolutionReviewState,
+  EvolutionTriggerRecord,
+} from "../schema.ts";
+import {
+  REVIEW_STATES,
+  hasConcreteRepoProposalChanges,
+  listDirectoryNames,
+  parseEvosCase,
+  parseKnowledgeRecord,
+  parseRepoProposal,
+  parseReviewCandidate,
+  parseTrigger,
+  sanitizeId,
+  sanitizeStorageId,
+  sanitizeText,
+  uniqueSorted,
+  validateEvolutionKnowledgeRecord,
+  validateEvolutionRepoProposal,
+} from "../shared.ts";
+
+const REVIEW_LOCK_WAIT_MS = 2_000;
+const REVIEW_LOCK_RETRY_MS = 10;
+
+export async function withEvolutionReviewDecisionLock<T>(
+  input: {
+    homeDir: string;
+    kind: "learning" | "knowledge" | "repo-proposal";
+    itemId: string;
+  },
+  operation: () => Promise<T>,
+): Promise<T> {
+  const paths = resolveEvoDevPaths(input.homeDir);
+  const key = createHash("sha256").update(`${input.kind}\0${input.itemId}`).digest("hex");
+  const lockPath = join(paths.stateDir, "evolution", ".review-locks", `${key}.lock`);
+  const deadline = Date.now() + REVIEW_LOCK_WAIT_MS;
+  await mkdir(dirname(lockPath), { recursive: true });
+
+  while (true) {
+    try {
+      await writeFile(
+        lockPath,
+        `${JSON.stringify({ version: 1, kind: input.kind, acquiredAt: new Date().toISOString(), pid: process.pid })}\n`,
+        { encoding: "utf8", flag: "wx" },
+      );
+      break;
+    } catch (error) {
+      if (!isAlreadyExistsError(error)) throw error;
+      if (Date.now() >= deadline) throw new Error("Review decision lock is busy or stale.");
+      await sleep(REVIEW_LOCK_RETRY_MS);
+    }
+  }
+
+  try {
+    return await operation();
+  } finally {
+    await rm(lockPath, { force: true });
+  }
+}
+
+export async function readEvolutionReviewSnapshot(input: {
+  homeDir: string;
+  projectKey?: string;
+}): Promise<EvolutionReviewSnapshot> {
+  const paths = resolveEvoDevPaths(input.homeDir);
+  const projectKeys =
+    input.projectKey === undefined
+      ? await listEvolutionReviewProjectKeys(paths)
+      : [sanitizeStorageId("projectKey", input.projectKey)];
+  const knowledgeRecords: EvolutionKnowledgeRecord[] = [];
+  const evosCases: EvolutionEvosCase[] = [];
+  const repoProposals: EvolutionRepoProposal[] = [];
+  const reviewCandidates: EvolutionReviewCandidate[] = [];
+  const triggers: EvolutionTriggerRecord[] = [];
+
+  for (const projectKey of projectKeys) {
+    const resolved = resolveEvolutionPaths({
+      homeDir: input.homeDir,
+      projectKey,
+      runId: "review",
+    });
+    knowledgeRecords.push(
+      ...(await readJsonFiles(resolved.knowledgeRecordsDir, parseKnowledgeRecord)),
+    );
+    evosCases.push(...(await readJsonFiles(resolved.evosCasesProjectDir, parseEvosCase)));
+  }
+
+  const evolutionStateDir = join(paths.stateDir, "evolution");
+  const stateProjectKeys =
+    input.projectKey === undefined
+      ? await listDirectoryNames(evolutionStateDir)
+      : [sanitizeStorageId("projectKey", input.projectKey)];
+  for (const projectKey of stateProjectKeys) {
+    const projectStateDir = join(evolutionStateDir, projectKey);
+    const runIds = await listDirectoryNames(projectStateDir);
+    for (const runId of runIds) {
+      const proposalsDir = join(projectStateDir, runId, "proposals");
+      const reviewCandidatesDir = join(projectStateDir, runId, "review-candidates");
+      repoProposals.push(...(await readJsonFiles(proposalsDir, parseRepoProposal)));
+      reviewCandidates.push(...(await readJsonFiles(reviewCandidatesDir, parseReviewCandidate)));
+      triggers.push(
+        ...(await readJsonFiles(join(projectStateDir, runId, "triggers"), parseTrigger)),
+      );
+    }
+  }
+
+  return {
+    projectKey:
+      input.projectKey === undefined ? null : sanitizeStorageId("projectKey", input.projectKey),
+    knowledgeRecords,
+    evosCases,
+    repoProposals,
+    reviewCandidates,
+    triggers,
+  };
+}
+
+async function listEvolutionReviewProjectKeys(
+  paths: ReturnType<typeof resolveEvoDevPaths>,
+): Promise<string[]> {
+  const legacyKnowledgeProjectKeys = (
+    await Promise.all(
+      (
+        await listDirectoryNames(paths.knowledgeDir)
+      ).map(async (projectKey) =>
+        (await pathExists(join(paths.knowledgeDir, projectKey, "records"))) ? projectKey : null,
+      ),
+    )
+  ).filter((projectKey): projectKey is string => projectKey !== null);
+  return uniqueSorted([
+    ...legacyKnowledgeProjectKeys,
+    ...(await listDirectoryNames(paths.evosCasesDir)),
+  ]);
+}
+
+export async function listEvolutionKnowledgeRecords(input: {
+  homeDir: string;
+  projectKey?: string;
+}): Promise<EvolutionKnowledgeRecord[]> {
+  return (
+    await readEvolutionReviewSnapshot({
+      homeDir: input.homeDir,
+      projectKey: input.projectKey,
+    })
+  ).knowledgeRecords.sort((left, right) => left.id.localeCompare(right.id));
+}
+
+export async function listEvolutionKnowledgeReviewHistory(input: {
+  homeDir: string;
+  projectKey?: string;
+}): Promise<EvolutionKnowledgeReviewHistoryRecord[]> {
+  const paths = resolveEvoDevPaths(input.homeDir);
+  const projectKeys =
+    input.projectKey === undefined
+      ? await listDirectoryNames(join(paths.stateDir, "evolution"))
+      : [sanitizeStorageId("projectKey", input.projectKey)];
+  const records: EvolutionKnowledgeReviewHistoryRecord[] = [];
+
+  for (const projectKey of projectKeys) {
+    const historyPath = resolveEvolutionPaths({
+      homeDir: input.homeDir,
+      projectKey,
+      runId: "review",
+    }).knowledgeReviewHistoryPath;
+    if (!(await pathExists(historyPath))) continue;
+    for (const line of (await readFile(historyPath, "utf8")).split("\n")) {
+      if (line.trim() === "") continue;
+      records.push(parseKnowledgeReviewHistoryRecord(JSON.parse(line) as unknown));
+    }
+  }
+
+  return records.sort((left, right) => {
+    const changedAt = left.changedAt.localeCompare(right.changedAt);
+    if (changedAt !== 0) return changedAt;
+    return left.knowledgeId.localeCompare(right.knowledgeId);
+  });
+}
+
+export async function listEvolutionEvosCases(input: {
+  homeDir: string;
+  projectKey?: string;
+  roleId?: string;
+  reviewStates?: EvolutionReviewState[];
+}): Promise<EvolutionEvosCaseQueryResult> {
+  const paths = resolveEvoDevPaths(input.homeDir);
+  const projectKeys =
+    input.projectKey === undefined
+      ? await listDirectoryNames(paths.evosCasesDir)
+      : [sanitizeStorageId("projectKey", input.projectKey)];
+  const roleId = input.roleId === undefined ? null : sanitizeId(input.roleId);
+  const reviewStates = input.reviewStates ?? ["accepted", "auto-accepted"];
+  const cases: EvolutionEvosCase[] = [];
+  const warnings: string[] = [];
+
+  for (const projectKey of projectKeys) {
+    const projectDir = join(paths.evosCasesDir, projectKey);
+    if (!(await pathExists(projectDir))) continue;
+    const entries = await readdir(projectDir, { withFileTypes: true });
+    for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) {
+      if (!entry.isFile() || !entry.name.endsWith(".json")) continue;
+      const sourceLink = `cases/${projectKey}/${entry.name}`;
+      try {
+        const value = JSON.parse(await readFile(join(projectDir, entry.name), "utf8")) as unknown;
+        const evosCase = parseEvosCase(value);
+        if (!reviewStates.includes(evosCase.reviewState)) continue;
+        if (
+          roleId !== null &&
+          evosCase.roleTags.length > 0 &&
+          !evosCase.roleTags.includes(roleId)
+        ) {
+          continue;
+        }
+        cases.push(evosCase);
+      } catch {
+        warnings.push(
+          `Omitted unsafe or invalid evos case: ${sourceLink}. Run evodev knowledge lint.`,
+        );
+      }
+    }
+  }
+
+  return {
+    cases: cases.sort((left, right) => {
+      const project = left.projectKey.localeCompare(right.projectKey);
+      if (project !== 0) return project;
+      return left.id.localeCompare(right.id);
+    }),
+    warnings,
+  };
+}
+
+export async function readEvolutionKnowledgeRecordById(input: {
+  homeDir: string;
+  knowledgeId: string;
+  projectKey?: string;
+}): Promise<EvolutionKnowledgeRecord> {
+  const matches = (await listEvolutionKnowledgeRecords(input)).filter(
+    (record) => record.id === input.knowledgeId,
+  );
+  if (matches.length === 0) throw new Error(`Knowledge record not found: ${input.knowledgeId}`);
+  if (matches.length > 1) {
+    throw new Error(`Knowledge record id is ambiguous across projects: ${input.knowledgeId}`);
+  }
+  return matches[0] as EvolutionKnowledgeRecord;
+}
+
+export async function updateEvolutionKnowledgeReviewState(input: {
+  homeDir: string;
+  knowledgeId: string;
+  projectKey?: string;
+  reviewState: "accepted" | "rejected" | "deferred";
+  expectedReviewState?: EvolutionReviewState;
+  now?: string | Date;
+}): Promise<{ path: string; record: EvolutionKnowledgeRecord; changed: boolean }> {
+  return await withEvolutionReviewDecisionLock(
+    { homeDir: input.homeDir, kind: "knowledge", itemId: input.knowledgeId },
+    async () => {
+      const record = await readEvolutionKnowledgeRecordById(input);
+      if (
+        input.expectedReviewState !== undefined &&
+        record.reviewState !== input.expectedReviewState
+      ) {
+        throw new Error("Knowledge review state changed before the decision was applied.");
+      }
+      const paths = resolveEvolutionPaths({
+        homeDir: input.homeDir,
+        projectKey: record.projectKey,
+        runId: "review",
+      });
+      const path = join(paths.knowledgeRecordsDir, `${record.id}.json`);
+      if (record.reviewState === input.reviewState) return { path, record, changed: false };
+
+      const changedAt = normalizeTimestamp(input.now);
+      const next: EvolutionKnowledgeRecord = {
+        ...record,
+        reviewState: input.reviewState,
+        authority: input.reviewState === "accepted" ? "reviewed" : "contextual",
+        runtime: {
+          ...record.runtime,
+          // Legacy JSON review records are not active OKF concepts.
+          canLoad: false,
+          hardBlocking: false,
+        },
+      };
+      validateEvolutionKnowledgeRecord(next);
+      await writeJson(path, next, { overwrite: true });
+      const allProjectKnowledgeRecords = await readJsonFiles(
+        paths.knowledgeRecordsDir,
+        parseKnowledgeRecord,
+      );
+      await writeJson(
+        paths.knowledgeIndexPath,
+        {
+          version: 1,
+          kind: "evolution-knowledge-index",
+          projectKey: next.projectKey,
+          updatedAt: changedAt,
+          records: allProjectKnowledgeRecords.map((item) => ({
+            id: item.id,
+            kind: item.kind,
+            title: item.title,
+            roleTags: item.roleTags,
+            tags: item.tags,
+            reviewState: item.reviewState,
+            authority: item.authority,
+            confidence: item.confidence,
+            runtime: item.runtime,
+          })),
+        },
+        { overwrite: true },
+      );
+      const history: EvolutionKnowledgeReviewHistoryRecord = {
+        version: 1,
+        kind: "evolution-knowledge-review-state-changed",
+        knowledgeId: next.id,
+        projectKey: next.projectKey,
+        roleTags: uniqueSorted(next.roleTags),
+        artifactCreatedAt: next.provenance.createdAt,
+        previousReviewState: record.reviewState,
+        nextReviewState: next.reviewState,
+        changedAt,
+        metadataOnly: true,
+      };
+      await mkdir(dirname(paths.knowledgeReviewHistoryPath), { recursive: true });
+      await writeFile(paths.knowledgeReviewHistoryPath, `${JSON.stringify(history)}\n`, {
+        encoding: "utf8",
+        flag: "a",
+      });
+      return { path, record: next, changed: true };
+    },
+  );
+}
+
+export async function readEvolutionRepoProposalById(input: {
+  homeDir: string;
+  proposalId: string;
+  projectKey?: string;
+}): Promise<EvolutionRepoProposal> {
+  const matches = (
+    await readEvolutionReviewSnapshot({
+      homeDir: input.homeDir,
+      projectKey: input.projectKey,
+    })
+  ).repoProposals.filter((proposal) => proposal.id === input.proposalId);
+  if (matches.length === 0) throw new Error(`Repo proposal not found: ${input.proposalId}`);
+  if (matches.length > 1) {
+    throw new Error(`Repo proposal id is ambiguous across runs: ${input.proposalId}`);
+  }
+  return matches[0] as EvolutionRepoProposal;
+}
+
+export async function updateEvolutionRepoProposalReviewState(input: {
+  homeDir: string;
+  proposalId: string;
+  projectKey?: string;
+  reviewState: "accepted" | "rejected" | "deferred";
+  expectedReviewState?: EvolutionRepoProposal["reviewState"];
+  reason?: string;
+  now?: string | Date;
+}): Promise<{ path: string; record: EvolutionRepoProposal; changed: boolean }> {
+  return await withEvolutionReviewDecisionLock(
+    { homeDir: input.homeDir, kind: "repo-proposal", itemId: input.proposalId },
+    async () => {
+      const record = await readEvolutionRepoProposalById(input);
+      if (!hasConcreteRepoProposalChanges(record)) {
+        throw new Error("Repo proposal has no concrete repository changes to review.");
+      }
+      const reason = input.reason?.trim();
+      if (input.reviewState === "rejected" && reason === undefined) {
+        throw new Error("Rejecting a repo proposal requires a reason.");
+      }
+      if (reason !== undefined && (reason === "" || reason.length > 500)) {
+        throw new Error("Repo proposal review reason must be 1-500 characters.");
+      }
+      if (
+        input.expectedReviewState !== undefined &&
+        record.reviewState !== input.expectedReviewState
+      ) {
+        throw new Error("Repo proposal review state changed before the decision was applied.");
+      }
+      const paths = resolveEvolutionPaths({
+        homeDir: input.homeDir,
+        projectKey: record.projectKey,
+        runId: record.provenance.runId,
+      });
+      const path = join(paths.repoProposalsDir, `${record.id}.json`);
+      if (record.reviewState === input.reviewState) return { path, record, changed: false };
+
+      const changedAt = normalizeTimestamp(input.now);
+      const next: EvolutionRepoProposal = {
+        ...record,
+        reviewState: input.reviewState,
+        reviewStateChangedAt: changedAt,
+        lastDecision: {
+          state: input.reviewState,
+          reason: reason === undefined ? null : sanitizeText(reason),
+          decidedAt: changedAt,
+        },
+      };
+      validateEvolutionRepoProposal(next);
+      await writeRepoProposalAndIndex({ homeDir: input.homeDir, proposal: next, changedAt });
+      return { path, record: next, changed: true };
+    },
+  );
+}
+
+export async function markEvolutionRepoProposalApplied(input: {
+  homeDir: string;
+  proposalId: string;
+  projectKey?: string;
+  expectedReviewState?: EvolutionRepoProposal["reviewState"];
+  now?: string | Date;
+}): Promise<{ path: string; record: EvolutionRepoProposal; changed: boolean }> {
+  return await withEvolutionReviewDecisionLock(
+    { homeDir: input.homeDir, kind: "repo-proposal", itemId: input.proposalId },
+    async () => {
+      const record = await readEvolutionRepoProposalById(input);
+      if (!hasConcreteRepoProposalChanges(record)) {
+        throw new Error("Repo proposal has no concrete repository changes to apply.");
+      }
+      if (
+        input.expectedReviewState !== undefined &&
+        record.reviewState !== input.expectedReviewState
+      ) {
+        throw new Error("Repo proposal review state changed before it was marked applied.");
+      }
+      const paths = resolveEvolutionPaths({
+        homeDir: input.homeDir,
+        projectKey: record.projectKey,
+        runId: record.provenance.runId,
+      });
+      const path = join(paths.repoProposalsDir, `${record.id}.json`);
+      if (record.reviewState === "applied") return { path, record, changed: false };
+      if (record.reviewState !== "accepted") {
+        throw new Error("Only accepted repo proposals can be marked applied.");
+      }
+
+      const changedAt = normalizeTimestamp(input.now);
+      const next: EvolutionRepoProposal = {
+        ...record,
+        reviewState: "applied",
+        reviewStateChangedAt: changedAt,
+      };
+      validateEvolutionRepoProposal(next);
+      await writeRepoProposalAndIndex({ homeDir: input.homeDir, proposal: next, changedAt });
+      return { path, record: next, changed: true };
+    },
+  );
+}
+
+async function writeRepoProposalAndIndex(input: {
+  homeDir: string;
+  proposal: EvolutionRepoProposal;
+  changedAt: string;
+}): Promise<void> {
+  const paths = resolveEvolutionPaths({
+    homeDir: input.homeDir,
+    projectKey: input.proposal.projectKey,
+    runId: input.proposal.provenance.runId,
+  });
+  await writeJson(join(paths.repoProposalsDir, `${input.proposal.id}.json`), input.proposal, {
+    overwrite: true,
+  });
+  const allRunProposals = await readJsonFiles(paths.repoProposalsDir, parseRepoProposal);
+  await writeJson(
+    paths.repoProposalsIndexPath,
+    {
+      schemaVersion: 1,
+      projectKey: input.proposal.projectKey,
+      runId: input.proposal.provenance.runId,
+      updatedAt: input.changedAt,
+      proposals: allRunProposals.map((proposal) => ({
+        id: proposal.id,
+        kind: proposal.kind,
+        title: proposal.title,
+        reviewState: proposal.reviewState,
+      })),
+    },
+    { overwrite: true },
+  );
+}
+
+function parseKnowledgeReviewHistoryRecord(value: unknown): EvolutionKnowledgeReviewHistoryRecord {
+  if (!isRecord(value)) throw new Error("Knowledge review history record must be an object.");
+  const allowedKeys = new Set([
+    "version",
+    "kind",
+    "knowledgeId",
+    "projectKey",
+    "roleTags",
+    "artifactCreatedAt",
+    "previousReviewState",
+    "nextReviewState",
+    "changedAt",
+    "metadataOnly",
+  ]);
+  if (Object.keys(value).some((key) => !allowedKeys.has(key))) {
+    throw new Error("Knowledge review history record contains unsupported fields.");
+  }
+  if (
+    value.version !== 1 ||
+    value.kind !== "evolution-knowledge-review-state-changed" ||
+    typeof value.knowledgeId !== "string" ||
+    typeof value.projectKey !== "string" ||
+    !Array.isArray(value.roleTags) ||
+    !value.roleTags.every((roleId) => typeof roleId === "string") ||
+    typeof value.artifactCreatedAt !== "string" ||
+    typeof value.previousReviewState !== "string" ||
+    !REVIEW_STATES.includes(value.previousReviewState as EvolutionReviewState) ||
+    typeof value.nextReviewState !== "string" ||
+    !REVIEW_STATES.includes(value.nextReviewState as EvolutionReviewState) ||
+    typeof value.changedAt !== "string" ||
+    value.metadataOnly !== true
+  ) {
+    throw new Error("Invalid knowledge review history record.");
+  }
+  normalizeTimestamp(value.artifactCreatedAt);
+  normalizeTimestamp(value.changedAt);
+  return {
+    version: 1,
+    kind: "evolution-knowledge-review-state-changed",
+    knowledgeId: sanitizeId(value.knowledgeId),
+    projectKey: sanitizeStorageId("projectKey", value.projectKey),
+    roleTags: uniqueSorted(value.roleTags.map(sanitizeId)),
+    artifactCreatedAt: value.artifactCreatedAt,
+    previousReviewState: value.previousReviewState as EvolutionReviewState,
+    nextReviewState: value.nextReviewState as EvolutionReviewState,
+    changedAt: value.changedAt,
+    metadataOnly: true,
+  };
+}
+
+function normalizeTimestamp(value: string | Date | undefined): string {
+  const date = value === undefined ? new Date() : value instanceof Date ? value : new Date(value);
+  if (Number.isNaN(date.getTime())) throw new Error("Invalid knowledge review history timestamp.");
+  return date.toISOString();
+}
+
+function isAlreadyExistsError(error: unknown): boolean {
+  return (
+    error instanceof Error && "code" in error && (error as NodeJS.ErrnoException).code === "EEXIST"
+  );
+}
+
+async function sleep(milliseconds: number): Promise<void> {
+  await new Promise((resolve) => setTimeout(resolve, milliseconds));
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}

package/src/evolution/control/index.ts

@@ -0,0 +1,20 @@
+export {
+  formatEvolutionProcessResult,
+  formatEvolutionReviewSnapshot,
+} from "../formatters.ts";
+export { processEvolutionTriggers } from "../processor/index.ts";
+export { readEvolutionReviewSnapshot } from "../candidates/index.ts";
+export {
+  listEvolutionTriggers,
+  listSegmentEvolutionTriggers,
+} from "../triggers/index.ts";
+export type {
+  EvolutionProcessInput,
+  EvolutionProcessProgress,
+  EvolutionProcessResult,
+  EvolutionReviewSnapshot,
+  EvolutionTriggerListInput,
+  EvolutionTriggerRecord,
+  SegmentEvolutionTriggerListInput,
+  SegmentEvolutionTriggerRecord,
+} from "../schema.ts";