@drewpayment/mink 0.1.0

package/src/core/reflection.ts

@@ -0,0 +1,256 @@
+import type { LearningMemory, SectionName, ReflectionResult } from "../types/learning-memory";
+import { serializeLearningMemory } from "./learning-memory";
+import { estimateTokens } from "./token-estimate";
+
+// Trim order: Decision Log first → Key Learnings → User Preferences → Do-Not-Repeat last
+const TRIM_ORDER: SectionName[] = [
+  "Decision Log",
+  "Key Learnings",
+  "User Preferences",
+  "Do-Not-Repeat",
+];
+
+function normalizeWhitespace(s: string): string {
+  return s.replace(/\s+/g, " ").trim();
+}
+
+/**
+ * Extract the quoted pattern from a Do-Not-Repeat entry, e.g.
+ * `Don't use "var" in code` → `var`
+ * Only matches double-quoted strings to avoid matching apostrophes in contractions.
+ * Returns null if no double-quoted pattern found.
+ */
+function extractQuotedPattern(entry: string): string | null {
+  const m = entry.match(/"([^"]+)"/);
+  return m ? m[1] : null;
+}
+
+/**
+ * Extract date from an entry in the form `[YYYY-MM-DD]`
+ * Returns the date string or null.
+ */
+function extractDate(entry: string): string | null {
+  const m = entry.match(/\[(\d{4}-\d{2}-\d{2})\]/);
+  return m ? m[1] : null;
+}
+
+function deepCopy(mem: LearningMemory): LearningMemory {
+  return {
+    projectName: mem.projectName,
+    sections: {
+      "User Preferences": [...mem.sections["User Preferences"]],
+      "Key Learnings": [...mem.sections["Key Learnings"]],
+      "Do-Not-Repeat": [...mem.sections["Do-Not-Repeat"]],
+      "Decision Log": [...mem.sections["Decision Log"]],
+    },
+  };
+}
+
+/**
+ * Merge duplicates within each section.
+ * - Exact duplicates (normalized whitespace) → keep one
+ * - Do-Not-Repeat: entries sharing same quoted pattern → merge, keep newer date
+ * Returns a new LearningMemory (does not mutate input).
+ */
+export function mergeDuplicates(mem: LearningMemory): LearningMemory {
+  const result = deepCopy(mem);
+
+  const sectionNames: SectionName[] = [
+    "User Preferences",
+    "Key Learnings",
+    "Do-Not-Repeat",
+    "Decision Log",
+  ];
+
+  for (const section of sectionNames) {
+    const entries = result.sections[section];
+
+    if (section === "Do-Not-Repeat") {
+      // First pass: group by quoted pattern (where it exists)
+      const byQuotedPattern = new Map<string, string[]>();
+      const noPattern: string[] = [];
+
+      for (const entry of entries) {
+        const qp = extractQuotedPattern(entry);
+        if (qp !== null) {
+          if (!byQuotedPattern.has(qp)) {
+            byQuotedPattern.set(qp, []);
+          }
+          byQuotedPattern.get(qp)!.push(entry);
+        } else {
+          noPattern.push(entry);
+        }
+      }
+
+      const merged: string[] = [];
+
+      // For each quoted pattern group, keep the one with the newer date (or last entry)
+      for (const [, group] of byQuotedPattern) {
+        if (group.length === 1) {
+          merged.push(group[0]);
+        } else {
+          // Find the entry with the newest date
+          let best = group[0];
+          let bestDate = extractDate(group[0]);
+          for (let i = 1; i < group.length; i++) {
+            const d = extractDate(group[i]);
+            if (d !== null && (bestDate === null || d > bestDate)) {
+              best = group[i];
+              bestDate = d;
+            } else if (d === null && bestDate === null) {
+              // No dates — keep last (newer = later in list)
+              best = group[i];
+            }
+          }
+          merged.push(best);
+        }
+      }
+
+      // For entries without quoted patterns, deduplicate by normalized whitespace
+      const seenNoPattern = new Set<string>();
+      for (const entry of noPattern) {
+        const norm = normalizeWhitespace(entry);
+        if (!seenNoPattern.has(norm)) {
+          seenNoPattern.add(norm);
+          merged.push(entry);
+        }
+      }
+
+      result.sections[section] = merged;
+    } else {
+      // Standard deduplication: normalize whitespace and deduplicate
+      const seen = new Set<string>();
+      const deduped: string[] = [];
+      for (const entry of entries) {
+        const norm = normalizeWhitespace(entry);
+        if (!seen.has(norm)) {
+          seen.add(norm);
+          deduped.push(entry);
+        }
+      }
+      result.sections[section] = deduped;
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Trim oldest entries from sections in the given order.
+ * Trim order: Decision Log → Key Learnings → User Preferences → Do-Not-Repeat
+ * Within each section, remove oldest (first) entries.
+ * Returns a new LearningMemory (does not mutate input).
+ */
+export function trimOldest(mem: LearningMemory, trimCount: number): LearningMemory {
+  if (trimCount <= 0) {
+    return deepCopy(mem);
+  }
+
+  const result = deepCopy(mem);
+  let remaining = trimCount;
+
+  for (const section of TRIM_ORDER) {
+    if (remaining <= 0) break;
+    const entries = result.sections[section];
+    const toRemove = Math.min(remaining, entries.length);
+    result.sections[section] = entries.slice(toRemove);
+    remaining -= toRemove;
+  }
+
+  return result;
+}
+
+/**
+ * Reflect memory against a token budget: merge duplicates then trim oldest until within budget.
+ * If budget <= 0, skip pruning.
+ */
+export function reflectMemory(
+  mem: LearningMemory,
+  tokenBudget: number
+): { memory: LearningMemory; result: ReflectionResult } {
+  const serialized = serializeLearningMemory(mem);
+  const beforeTokens = estimateTokens(serialized, "learning-memory.md");
+
+  if (tokenBudget <= 0) {
+    return {
+      memory: deepCopy(mem),
+      result: {
+        beforeTokens,
+        afterTokens: beforeTokens,
+        mergedCount: 0,
+        trimmedCount: 0,
+        withinBudget: true,
+      },
+    };
+  }
+
+  // Step 1: Always merge duplicates (regardless of budget)
+  const beforeMergeCount = countEntries(mem);
+  const afterMerge = mergeDuplicates(mem);
+  const afterMergeCount = countEntries(afterMerge);
+  const mergedCount = beforeMergeCount - afterMergeCount;
+
+  const afterMergeSerialized = serializeLearningMemory(afterMerge);
+  const afterMergeTokens = estimateTokens(afterMergeSerialized, "learning-memory.md");
+
+  if (afterMergeTokens <= tokenBudget) {
+    return {
+      memory: afterMerge,
+      result: {
+        beforeTokens,
+        afterTokens: afterMergeTokens,
+        mergedCount,
+        trimmedCount: 0,
+        withinBudget: true,
+      },
+    };
+  }
+
+  // Step 2: Trim oldest one at a time until within budget or empty
+  let current = afterMerge;
+  let trimmedCount = 0;
+
+  while (true) {
+    const currentSerialized = serializeLearningMemory(current);
+    const currentTokens = estimateTokens(currentSerialized, "learning-memory.md");
+
+    if (currentTokens <= tokenBudget) {
+      return {
+        memory: current,
+        result: {
+          beforeTokens,
+          afterTokens: currentTokens,
+          mergedCount,
+          trimmedCount,
+          withinBudget: true,
+        },
+      };
+    }
+
+    const total = countEntries(current);
+    if (total === 0) {
+      return {
+        memory: current,
+        result: {
+          beforeTokens,
+          afterTokens: currentTokens,
+          mergedCount,
+          trimmedCount,
+          withinBudget: false,
+        },
+      };
+    }
+
+    current = trimOldest(current, 1);
+    trimmedCount += 1;
+  }
+}
+
+function countEntries(mem: LearningMemory): number {
+  return (
+    mem.sections["User Preferences"].length +
+    mem.sections["Key Learnings"].length +
+    mem.sections["Do-Not-Repeat"].length +
+    mem.sections["Decision Log"].length
+  );
+}

package/src/core/scanner.ts

@@ -0,0 +1,99 @@
+import { readdirSync, statSync } from "fs";
+import { join, relative } from "path";
+import type { ScannedFile, ProjectConfig } from "../types/file-index";
+import { safeReadJson } from "./fs-utils";
+
+export const DEFAULT_EXCLUDES: string[] = [
+  "node_modules", "vendor", ".venv", "venv", "__pycache__",
+  "bower_components", ".yarn", ".pnp",
+  "dist", "build", "out", ".next", ".nuxt", ".svelte-kit",
+  ".turbo", ".vercel", ".output",
+  "coverage", ".nyc_output",
+  ".git", ".hg", ".svn",
+  "package-lock.json", "bun.lock", "yarn.lock",
+  "pnpm-lock.yaml", "Gemfile.lock", "poetry.lock", "composer.lock",
+  "*.min.js", "*.min.css", "*.map",
+  "*.png", "*.jpg", "*.jpeg", "*.gif", "*.svg", "*.ico",
+  "*.woff", "*.woff2", "*.ttf", "*.eot",
+  "*.mp3", "*.mp4", "*.webm", "*.zip", "*.tar", "*.gz",
+  "*.pdf", "*.exe", "*.dll", "*.so", "*.dylib",
+  ".env", ".env.*",
+  ".mink",
+];
+
+const DEFAULT_MAX_FILES = 500;
+
+function matchesPattern(name: string, pattern: string): boolean {
+  if (pattern.includes("*")) {
+    // Glob: *.min.js -> match against basename
+    const regex = new RegExp(
+      "^" + pattern.replace(/\./g, "\\.").replace(/\*/g, ".*") + "$"
+    );
+    return regex.test(name);
+  }
+  // Exact match against name
+  return name === pattern;
+}
+
+function isExcluded(name: string, excludes: string[]): boolean {
+  return excludes.some((pattern) => matchesPattern(name, pattern));
+}
+
+function walkDirectory(
+  dir: string,
+  projectRoot: string,
+  excludes: string[],
+  results: ScannedFile[]
+): void {
+  let entries;
+  try {
+    entries = readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return; // Permission denied or other error — skip
+  }
+
+  for (const entry of entries) {
+    if (entry.isSymbolicLink()) continue;
+
+    if (entry.isDirectory()) {
+      if (isExcluded(entry.name, excludes)) continue;
+      walkDirectory(join(dir, entry.name), projectRoot, excludes, results);
+      continue;
+    }
+
+    if (entry.isFile()) {
+      if (isExcluded(entry.name, excludes)) continue;
+      try {
+        const fullPath = join(dir, entry.name);
+        const stat = statSync(fullPath);
+        results.push({
+          relativePath: relative(projectRoot, fullPath),
+          mtimeMs: stat.mtimeMs,
+        });
+      } catch {
+        // stat failed — skip
+      }
+    }
+  }
+}
+
+export function loadConfig(configPath: string): ProjectConfig {
+  const raw = safeReadJson(configPath);
+  if (raw && typeof raw === "object") return raw as ProjectConfig;
+  return {};
+}
+
+export function getExcludes(config: ProjectConfig): string[] {
+  return [...DEFAULT_EXCLUDES, ...(config.excludePatterns ?? [])];
+}
+
+export function scanProject(
+  projectRoot: string,
+  excludes: string[],
+  maxFiles: number = DEFAULT_MAX_FILES
+): ScannedFile[] {
+  const results: ScannedFile[] = [];
+  walkDirectory(projectRoot, projectRoot, excludes, results);
+  results.sort((a, b) => b.mtimeMs - a.mtimeMs);
+  return results.slice(0, maxFiles);
+}

package/src/core/scheduler.ts

@@ -0,0 +1,352 @@
+import { parseCronExpression, nextRunAfter, isInCurrentPeriod } from "./cron-parser";
+import { getBuiltInTasks, getTaskById, executeTask } from "./task-registry";
+import { schedulerManifestPath } from "./paths";
+import { atomicWriteJson, safeReadJson } from "./fs-utils";
+import type {
+  SchedulerManifest,
+  TaskRunRecord,
+  DeadLetterEntry,
+  HealthStatus,
+  TaskStatus,
+} from "../types/scheduler";
+
+// ── Backoff ─────────────────────────────────────────────────────────────────
+
+export function calculateBackoffMs(
+  baseDelayMs: number,
+  attempt: number
+): number {
+  return baseDelayMs * Math.pow(2, attempt);
+}
+
+// ── Dead Letter Operations ──────────────────────────────────────────────────
+
+export function addToDeadLetter(
+  manifest: SchedulerManifest,
+  entry: DeadLetterEntry
+): void {
+  // Remove existing entry for same task if present
+  manifest.deadLetterQueue = manifest.deadLetterQueue.filter(
+    (e) => e.taskId !== entry.taskId
+  );
+  manifest.deadLetterQueue.push(entry);
+}
+
+export function removeFromDeadLetter(
+  manifest: SchedulerManifest,
+  taskId: string
+): DeadLetterEntry | undefined {
+  const idx = manifest.deadLetterQueue.findIndex((e) => e.taskId === taskId);
+  if (idx === -1) return undefined;
+  return manifest.deadLetterQueue.splice(idx, 1)[0];
+}
+
+export function listDeadLetterEntries(
+  manifest: SchedulerManifest
+): DeadLetterEntry[] {
+  return manifest.deadLetterQueue;
+}
+
+// ── Manifest Management ─────────────────────────────────────────────────────
+
+export function createInitialManifest(now: Date = new Date()): SchedulerManifest {
+  const tasks: TaskRunRecord[] = getBuiltInTasks().map((task) => {
+    const schedule = parseCronExpression(task.schedule);
+    return {
+      taskId: task.id,
+      lastRunAt: null,
+      lastSuccessAt: null,
+      lastFailureAt: null,
+      nextRunAt: nextRunAfter(schedule, now).toISOString(),
+      status: "idle" as TaskStatus,
+      consecutiveFailures: 0,
+      currentAttempt: 0,
+    };
+  });
+
+  return {
+    tasks,
+    deadLetterQueue: [],
+    lastHeartbeat: now.toISOString(),
+  };
+}
+
+export function loadManifest(cwd: string): SchedulerManifest | null {
+  const raw = safeReadJson(schedulerManifestPath(cwd));
+  if (
+    raw &&
+    typeof raw === "object" &&
+    "tasks" in (raw as object) &&
+    "deadLetterQueue" in (raw as object)
+  ) {
+    return raw as SchedulerManifest;
+  }
+  return null;
+}
+
+export function saveManifest(cwd: string, manifest: SchedulerManifest): void {
+  atomicWriteJson(schedulerManifestPath(cwd), manifest);
+}
+
+function getOrCreateManifest(cwd: string, now: Date): SchedulerManifest {
+  const existing = loadManifest(cwd);
+  if (existing) return existing;
+  const fresh = createInitialManifest(now);
+  saveManifest(cwd, fresh);
+  return fresh;
+}
+
+// ── Crash Recovery ──────────────────────────────────────────────────────────
+
+export function recoverManifest(
+  manifest: SchedulerManifest,
+  now: Date
+): void {
+  for (const record of manifest.tasks) {
+    const task = getTaskById(record.taskId);
+    if (!task) continue;
+
+    const schedule = parseCronExpression(task.schedule);
+
+    if (record.status === "running") {
+      // Crashed during execution — treat as failure
+      record.status = "retrying";
+      record.currentAttempt++;
+      record.consecutiveFailures++;
+      record.lastFailureAt = now.toISOString();
+
+      if (record.currentAttempt >= task.retryPolicy.maxAttempts) {
+        record.status = "dead-lettered";
+        addToDeadLetter(manifest, {
+          taskId: record.taskId,
+          deadLetteredAt: now.toISOString(),
+          failureTimestamps: [now.toISOString()],
+          errorMessages: ["Daemon crashed during execution"],
+          attemptCount: record.currentAttempt,
+        });
+        record.currentAttempt = 0;
+      }
+    }
+
+    // For idle/retrying tasks, check if they need schedule recalculation
+    if (record.status === "idle" && record.lastRunAt) {
+      const lastRun = new Date(record.lastRunAt);
+      if (isInCurrentPeriod(schedule, lastRun, now)) {
+        // Already ran in current period — advance to next
+        record.nextRunAt = nextRunAfter(schedule, lastRun).toISOString();
+      } else {
+        // Missed the window — due now
+        record.nextRunAt = now.toISOString();
+      }
+    } else if (record.status === "idle" && !record.lastRunAt) {
+      // Never ran — recalculate next run
+      record.nextRunAt = nextRunAfter(schedule, now).toISOString();
+    }
+  }
+}
+
+// ── Scheduler ───────────────────────────────────────────────────────────────
+
+export interface Scheduler {
+  start(): void;
+  stop(): void;
+  runTask(taskId: string): Promise<void>;
+  getHealth(): HealthStatus;
+  getManifest(): SchedulerManifest;
+}
+
+export function createScheduler(
+  projectCwd: string,
+  options: {
+    tickMs?: number;
+    heartbeatMs?: number;
+    startedAt?: Date;
+  } = {}
+): Scheduler {
+  const tickMs = options.tickMs ?? 60_000;
+  const heartbeatMs = options.heartbeatMs ?? 30 * 60 * 1000;
+  const startedAt = options.startedAt ?? new Date();
+
+  let tickInterval: ReturnType<typeof setInterval> | null = null;
+  let heartbeatInterval: ReturnType<typeof setInterval> | null = null;
+  let manifest: SchedulerManifest;
+  let activeTasks: string[] = [];
+  let ticking = false;
+
+  // Initialize manifest
+  manifest = getOrCreateManifest(projectCwd, startedAt);
+  recoverManifest(manifest, startedAt);
+  saveManifest(projectCwd, manifest);
+
+  async function tick(): Promise<void> {
+    if (ticking) return; // Prevent overlapping ticks
+    ticking = true;
+
+    try {
+      const now = new Date();
+      const queue: string[] = [];
+
+      for (const record of manifest.tasks) {
+        const task = getTaskById(record.taskId);
+        if (!task || !task.enabled) continue;
+        if (record.status === "dead-lettered") continue;
+
+        if (record.status === "retrying") {
+          // Check if backoff delay has elapsed
+          const retryAfter =
+            new Date(record.lastFailureAt!).getTime() +
+            calculateBackoffMs(
+              task.retryPolicy.baseDelayMs,
+              record.currentAttempt - 1
+            );
+          if (now.getTime() >= retryAfter) {
+            queue.push(record.taskId);
+          }
+        } else if (
+          record.status === "idle" &&
+          now.getTime() >= new Date(record.nextRunAt).getTime()
+        ) {
+          queue.push(record.taskId);
+        }
+      }
+
+      // Execute sequentially, sorted by task ID for determinism
+      queue.sort();
+
+      for (const taskId of queue) {
+        await executeTaskWithRetry(taskId, now);
+      }
+    } finally {
+      ticking = false;
+    }
+  }
+
+  async function executeTaskWithRetry(
+    taskId: string,
+    now: Date
+  ): Promise<void> {
+    const task = getTaskById(taskId);
+    if (!task) return;
+
+    const record = manifest.tasks.find((r) => r.taskId === taskId);
+    if (!record) return;
+
+    record.status = "running";
+    record.lastRunAt = now.toISOString();
+    activeTasks.push(taskId);
+    saveManifest(projectCwd, manifest);
+
+    try {
+      await executeTask(taskId, projectCwd);
+
+      // Success
+      record.status = "idle";
+      record.lastSuccessAt = now.toISOString();
+      record.consecutiveFailures = 0;
+      record.currentAttempt = 0;
+      const schedule = parseCronExpression(task.schedule);
+      record.nextRunAt = nextRunAfter(schedule, now).toISOString();
+
+      console.log(`[mink] task ${taskId} completed successfully`);
+    } catch (err) {
+      const errorMsg =
+        err instanceof Error ? err.message : String(err);
+      record.currentAttempt++;
+      record.consecutiveFailures++;
+      record.lastFailureAt = now.toISOString();
+
+      console.error(`[mink] task ${taskId} failed: ${errorMsg}`);
+
+      if (record.currentAttempt >= task.retryPolicy.maxAttempts) {
+        record.status = "dead-lettered";
+        addToDeadLetter(manifest, {
+          taskId,
+          deadLetteredAt: now.toISOString(),
+          failureTimestamps: [now.toISOString()],
+          errorMessages: [errorMsg],
+          attemptCount: record.currentAttempt,
+        });
+        record.currentAttempt = 0;
+        console.error(
+          `[mink] task ${taskId} moved to dead letter queue after ${task.retryPolicy.maxAttempts} failures`
+        );
+      } else {
+        record.status = "retrying";
+        console.log(
+          `[mink] task ${taskId} will retry (attempt ${record.currentAttempt}/${task.retryPolicy.maxAttempts})`
+        );
+      }
+    } finally {
+      activeTasks = activeTasks.filter((id) => id !== taskId);
+      saveManifest(projectCwd, manifest);
+    }
+  }
+
+  function emitHeartbeat(): void {
+    manifest.lastHeartbeat = new Date().toISOString();
+    saveManifest(projectCwd, manifest);
+    console.log(`[mink] heartbeat at ${manifest.lastHeartbeat}`);
+  }
+
+  return {
+    start(): void {
+      tickInterval = setInterval(() => {
+        tick().catch((err) => {
+          console.error(`[mink] scheduler tick error: ${err}`);
+        });
+      }, tickMs);
+
+      heartbeatInterval = setInterval(emitHeartbeat, heartbeatMs);
+
+      // Emit initial heartbeat
+      emitHeartbeat();
+
+      // Run first tick immediately
+      tick().catch((err) => {
+        console.error(`[mink] scheduler initial tick error: ${err}`);
+      });
+
+      console.log("[mink] scheduler started");
+    },
+
+    stop(): void {
+      if (tickInterval) {
+        clearInterval(tickInterval);
+        tickInterval = null;
+      }
+      if (heartbeatInterval) {
+        clearInterval(heartbeatInterval);
+        heartbeatInterval = null;
+      }
+      console.log("[mink] scheduler stopped");
+    },
+
+    async runTask(taskId: string): Promise<void> {
+      const task = getTaskById(taskId);
+      if (!task) {
+        throw new Error(`Unknown task: ${taskId}`);
+      }
+      // Reload manifest to get latest state
+      const fresh = loadManifest(projectCwd);
+      if (fresh) manifest = fresh;
+
+      await executeTaskWithRetry(taskId, new Date());
+    },
+
+    getHealth(): HealthStatus {
+      return {
+        pid: process.pid,
+        startedAt: startedAt.toISOString(),
+        lastHeartbeatAt: manifest.lastHeartbeat,
+        uptimeMs: Date.now() - startedAt.getTime(),
+        activeTasks: [...activeTasks],
+        deadLetterCount: manifest.deadLetterQueue.length,
+        taskCount: manifest.tasks.length,
+      };
+    },
+
+    getManifest(): SchedulerManifest {
+      return manifest;
+    },
+  };
+}