boxsafe 1.0.0

package/core/loop/runValidation.ts

@@ -0,0 +1,25 @@
+import { waterfall } from '@core/loop/waterfall';
+import { Logger } from '@core/util/logger';
+
+const logger = Logger.createModuleLogger('RunValidation');
+
+type RunValidationArgs = {
+  execResult: any;
+  pathOutput: string;
+  signal?: AbortSignal;
+};
+
+export async function runValidation({ execResult, pathOutput, signal }: RunValidationArgs): Promise<any> {
+  if (signal?.aborted) throw new Error('Aborted');
+  try {
+    return await waterfall({
+      exec: execResult,
+      artifacts: {
+        outputFile: pathOutput,
+      },
+    });
+  } catch (err: any) {
+    logger.error(`Waterfall error: ${err?.message ?? err}`);
+    throw err;
+  }
+}

package/core/loop/tasks/AGENT-TASKS.md

@@ -0,0 +1,36 @@
+**Tasks module for BoxSafe — core/loop/tasks**
+
+Overview
+- The `TasksManager` provides a minimal, predictable mechanism to load a TODO file,
+  split it into small tasks, persist task artifacts and a lightweight runtime state
+  under `memo/state/tasks`.
+
+Design goals
+- Keep implementation tiny and synchronous-friendly (async IO only).
+- Avoid complex formats: accept either `- item` lines or paragraphs separated by
+  blank lines in the TODO file.
+- Persist each task as `memo/state/tasks/tasks/task_###.md` and keep state in
+  `memo/state/tasks/state.json` with `current` and `done[]` fields.
+
+How it is used
+- The `execLoop` integrates `TasksManager` when `boxsafe.config.json` contains
+  `project.todo` pointing to a TODO file. The loop will use the current task as
+  the agent prompt, and when the agent completes a task it marks it done and
+  moves to the next. Only when all tasks complete does the loop perform the
+  normal after-success flow (versioning, notes, etc.).
+
+Files
+- `index.ts` — exports `TasksManager` class.
+
+Behavior summary
+- `TasksManager.init()` creates `memo/state/tasks` and writes individual task
+  files and `state.json` when needed.
+- `getCurrentTask()` returns the current task prompt or `null` when finished.
+- `markCurrentDone()` marks current task as done and advances to the next.
+- `isFinished()` signals completion of all tasks.
+
+Notes
+- The module intentionally keeps parsing and behavior simple to be robust and
+  easy to audit. If you want more advanced scheduling, priorities or parallel
+  execution, we can extend it, but this initial version focuses on correctness
+  and minimal surface area.

package/core/loop/tasks/index.ts

@@ -0,0 +1,96 @@
+import fs from 'node:fs/promises';
+import fsSync from 'node:fs';
+import path from 'node:path';
+import { TASKS_STATE_DIR, TASKS_TASKS_DIR, TASKS_STATE_FILE } from '@core/paths/paths';
+
+export class TasksManager {
+  todoPath: string;
+  baseDir: string;
+  tasksDir: string;
+  statePath: string;
+  tasks: string[] = [];
+  state: { current: number; done: boolean[] } = { current: 0, done: [] };
+
+  constructor(todoPath: string, baseDir?: string) {
+    this.todoPath = path.resolve(todoPath);
+    this.baseDir = baseDir ? path.resolve(baseDir) : TASKS_STATE_DIR;
+    this.tasksDir = baseDir ? path.join(this.baseDir, 'tasks') : TASKS_TASKS_DIR;
+    this.statePath = baseDir ? path.join(this.baseDir, 'state.json') : TASKS_STATE_FILE;
+  }
+
+  async init() {
+    // ensure directories
+    await fs.mkdir(this.tasksDir, { recursive: true });
+
+    // load or create tasks
+    if (fsSync.existsSync(this.statePath)) {
+      try {
+        const raw = await fs.readFile(this.statePath, 'utf-8');
+        this.state = JSON.parse(raw);
+        // load tasks files if present
+        const files = await fs.readdir(this.tasksDir);
+        this.tasks = [];
+        for (const f of files.sort()) {
+          const content = await fs.readFile(path.join(this.tasksDir, f), 'utf-8');
+          this.tasks.push(content);
+        }
+      } catch (e) {
+        // fallback to re-scan todo
+        await this._prepareFromTodo();
+      }
+    } else {
+      await this._prepareFromTodo();
+    }
+  }
+
+  private async _prepareFromTodo() {
+    this.tasks = [];
+    this.state = { current: 0, done: [] };
+    if (!fsSync.existsSync(this.todoPath)) return;
+    const raw = await fs.readFile(this.todoPath, 'utf-8');
+    // parse simple list format: lines starting with '-' or split by double-newline
+    const lines = raw.split(/\r?\n/).map((l) => l.trim()).filter(Boolean);
+    let items: string[] = [];
+    const dashItems = lines.filter((l) => l.startsWith('- ')).map((l) => l.replace(/^-\s+/, ''));
+    if (dashItems.length > 0) items = dashItems;
+    else items = raw.split(/\n\s*\n/).map(s=>s.trim()).filter(Boolean);
+
+    // write tasks files
+    for (let i = 0; i < items.length; i++) {
+      const filename = `task_${String(i+1).padStart(3,'0')}.md`;
+      const item = items[i] ?? '';
+      await fs.writeFile(path.join(this.tasksDir, filename), item, 'utf-8');
+      this.tasks.push(item);
+      this.state.done.push(false);
+    }
+
+    await this._saveState();
+  }
+
+  private async _saveState() {
+    await fs.writeFile(this.statePath, JSON.stringify(this.state, null, 2), 'utf-8');
+  }
+
+  getCurrentTask(): string | null {
+    if (this.state.current >= this.tasks.length) return null;
+    return this.tasks[this.state.current] ?? null;
+  }
+
+  async markCurrentDone() {
+    if (this.state.current >= this.tasks.length) return;
+    this.state.done[this.state.current] = true;
+    // advance to next not-done
+    let idx = this.state.current + 1;
+    while (idx < this.tasks.length && this.state.done[idx]) idx++;
+    this.state.current = idx;
+    await this._saveState();
+  }
+
+  isFinished(): boolean {
+    return this.state.current >= this.tasks.length;
+  }
+
+  total(): number { return this.tasks.length; }
+}
+
+export default TasksManager;

package/core/loop/toolCalls.ts

@@ -0,0 +1,168 @@
+import { Logger } from '@core/util/logger';
+
+const logger = Logger.createModuleLogger('ToolCalls');
+
+export type ToolName = 'navigate' | 'versionControl';
+
+export type NavigateOp = 'list' | 'read' | 'write' | 'mkdir' | 'delete' | 'stat';
+
+export type NavigateToolCall = {
+  tool: 'navigate';
+  params: {
+    op: NavigateOp;
+    path?: string;
+    content?: string;
+    writeOptions?: { append?: boolean; createDirs?: boolean };
+    mkdirOptions?: { recursive?: boolean };
+    deleteOptions?: { recursive?: boolean };
+  };
+};
+
+export type VersionControlToolCall = {
+  tool: 'versionControl';
+  params: Record<string, unknown>;
+};
+
+export type ToolCall = NavigateToolCall | VersionControlToolCall;
+
+export type ToolCallParseError = {
+  ok: false;
+  error: string;
+  fence?: string;
+};
+
+export type ToolCallParseResult = {
+  ok: true;
+  calls: ToolCall[];
+  errors: ToolCallParseError[];
+};
+
+function isPlainObject(v: unknown): v is Record<string, unknown> {
+  return Boolean(v) && typeof v === 'object' && !Array.isArray(v);
+}
+
+function isParseError(v: ToolCall | ToolCallParseError): v is ToolCallParseError {
+  return isPlainObject(v) && (v as any).ok === false;
+}
+
+function isNavigateOp(v: unknown): v is NavigateOp {
+  return (
+    v === 'list' ||
+    v === 'read' ||
+    v === 'write' ||
+    v === 'mkdir' ||
+    v === 'delete' ||
+    v === 'stat'
+  );
+}
+
+function parseOneToolCall(obj: unknown): ToolCall | ToolCallParseError {
+  if (!isPlainObject(obj)) return { ok: false, error: 'tool call must be an object' };
+
+  const tool = obj.tool;
+  if (tool !== 'navigate' && tool !== 'versionControl') {
+    return { ok: false, error: `unknown tool: ${String(tool)}` };
+  }
+
+  const paramsRaw = obj.params;
+  const params = isPlainObject(paramsRaw) ? paramsRaw : {};
+
+  if (tool === 'navigate') {
+    const op = params.op;
+    if (!isNavigateOp(op)) {
+      return { ok: false, error: `navigate.op must be one of list|read|write|mkdir|delete|stat (got: ${String(op)})` };
+    }
+
+    const path = typeof params.path === 'string' ? params.path : undefined;
+    const content = typeof params.content === 'string' ? params.content : undefined;
+
+    if ((op === 'read' || op === 'write' || op === 'mkdir' || op === 'delete' || op === 'stat') && !path) {
+      return { ok: false, error: `navigate.op=${op} requires params.path` };
+    }
+
+    if (op === 'write' && content === undefined) {
+      return { ok: false, error: `navigate.op=write requires params.content` };
+    }
+
+    const writeOptions = isPlainObject(params.writeOptions) ? params.writeOptions : undefined;
+    const mkdirOptions = isPlainObject(params.mkdirOptions) ? params.mkdirOptions : undefined;
+    const deleteOptions = isPlainObject(params.deleteOptions) ? params.deleteOptions : undefined;
+
+    return {
+      tool: 'navigate',
+      params: {
+        op,
+        ...(path ? { path } : {}),
+        ...(content !== undefined ? { content } : {}),
+        ...(writeOptions
+          ? {
+              writeOptions: {
+                ...(typeof writeOptions.append === 'boolean' ? { append: writeOptions.append } : {}),
+                ...(typeof writeOptions.createDirs === 'boolean' ? { createDirs: writeOptions.createDirs } : {}),
+              },
+            }
+          : {}),
+        ...(mkdirOptions
+          ? {
+              mkdirOptions: {
+                ...(typeof mkdirOptions.recursive === 'boolean' ? { recursive: mkdirOptions.recursive } : {}),
+              },
+            }
+          : {}),
+        ...(deleteOptions
+          ? {
+              deleteOptions: {
+                ...(typeof deleteOptions.recursive === 'boolean' ? { recursive: deleteOptions.recursive } : {}),
+              },
+            }
+          : {}),
+      },
+    };
+  }
+
+  return {
+    tool: 'versionControl',
+    params,
+  };
+}
+
+export function parseToolCallsFromMarkdown(markdown: string): ToolCallParseResult {
+  const calls: ToolCall[] = [];
+  const errors: ToolCallParseError[] = [];
+
+  logger.debug(`Parsing markdown for tool calls (${markdown.length} chars)`);
+
+  // FIXED: Changed regex to specifically target json-tool blocks
+  // The old regex /```(?:json|json-tool)\s*([\s\S]*?)\s*```/g was capturing the wrong group
+  const jsonFenceRe = /```json-tool\s*([\s\S]*?)\s*```/g;
+  let m: RegExpExecArray | null;
+
+  while ((m = jsonFenceRe.exec(markdown)) !== null) {
+    const fence = (m[1] ?? '').trim();
+    logger.debug(`Found potential tool call: ${fence.substring(0, 100)}...`);
+    
+    try {
+      const obj = JSON.parse(fence);
+      logger.debug(`JSON parsed successfully: ${JSON.stringify(obj)}`);
+      
+      const parsed = parseOneToolCall(obj);
+      
+      if (isParseError(parsed)) {
+        logger.warn(`Invalid tool call: ${parsed.error}`);
+        errors.push({ ...parsed, fence });
+      } else {
+        logger.info(`Valid tool call detected: ${parsed.tool}`);
+        calls.push(parsed);
+      }
+    } catch (e: any) {
+      logger.error(`JSON parse failed: ${e?.message ?? String(e)}`);
+      errors.push({ ok: false, error: `invalid JSON: ${e?.message ?? String(e)}`, fence });
+    }
+  }
+
+  logger.info(`Tool calls parsed: ${calls.length} valid, ${errors.length} errors`);
+  return { ok: true, calls, errors };
+}
+
+// Export helper functions for reuse
+export { parseOneToolCall, isParseError };

package/core/loop/toolDispatcher.ts

@@ -0,0 +1,146 @@
+import type { Navigator } from '@core/navigate';
+import { parseToolCallsFromMarkdown } from './toolCalls';
+import type { ToolCall } from './toolCalls';
+import type { TraceCtx } from './traceLogger';
+import { Logger } from '@core/util/logger';
+
+const logger = Logger.createModuleLogger('ToolDispatcher');
+
+type AnsiLike = { Cyan: string; Yellow: string; Red: string; Reset: string };
+
+type AttemptVersionControl = (opts: any) => Promise<any>;
+
+type TraceEmit = (event: string, ctx?: TraceCtx, data?: Record<string, unknown>) => Promise<void>;
+
+type DispatchArgs = {
+  markdown: string;
+  navigator: Navigator | null;
+  boxConfig: any;
+  ANSI: AnsiLike;
+  attemptVersionControl: AttemptVersionControl;
+  vcAutoPushConfig: boolean;
+  traceEmit?: TraceEmit;
+  traceCtx?: TraceCtx;
+};
+
+export async function dispatchToolCalls({
+  markdown,
+  navigator,
+  boxConfig,
+  ANSI,
+  attemptVersionControl,
+  vcAutoPushConfig,
+  traceEmit,
+  traceCtx,
+}: DispatchArgs): Promise<void> {
+  try {
+    logger.debug(`Dispatching tool calls from markdown (${markdown.length} chars)`);
+    logger.debug(`Navigator available: ${!!navigator}`);
+
+    const parsed = parseToolCallsFromMarkdown(markdown);
+    await traceEmit?.('toolcalls.parsed', traceCtx, {
+      calls: parsed.calls.length,
+      errors: parsed.errors.length,
+    });
+
+    logger.info(`Tool calls parsed: ${parsed.calls.length} valid, ${parsed.errors.length} errors`);
+    
+    for (const e of parsed.errors) {
+      logger.warn(`Invalid tool call: ${e.error}`);
+      await traceEmit?.('toolcall.invalid', traceCtx, { error: e.error });
+    }
+
+    if (parsed.calls.length === 0) {
+      logger.debug('No valid tool calls to execute');
+      return;
+    }
+
+    logger.info(`Executing ${parsed.calls.length} tool calls...`);
+    for (const call of parsed.calls) {
+      await executeOneToolCall(call);
+    }
+  } catch (toolErr: any) {
+    logger.error(`Tool parsing failed: ${toolErr?.message ?? toolErr}`);
+    await traceEmit?.('toolcalls.failed', traceCtx, { error: toolErr?.message ?? String(toolErr) });
+  }
+
+  async function executeOneToolCall(call: ToolCall): Promise<void> {
+    logger.debug(`Executing tool call: ${JSON.stringify(call)}`);
+    logger.info(`detected tool call: ${call.tool}`);
+    await traceEmit?.('toolcall.detected', traceCtx, { tool: call.tool });
+
+    if (call.tool === 'navigate') {
+      logger.debug(`Processing navigate tool call with op: ${call.params.op}`);
+      
+      if (!navigator) {
+        logger.warn(`navigate requested but navigator is not initialized`);
+        await traceEmit?.('toolcall.navigate.skipped', traceCtx, { reason: 'navigator_not_initialized' });
+        return;
+      }
+
+      const { op } = call.params;
+      let toolRes: any = null;
+      
+      try {
+        if (op === 'write') {
+          logger.debug(`Writing file: ${call.params.path}`);
+          toolRes = await navigator.writeFile(call.params.path!, call.params.content!, call.params.writeOptions);
+        } else if (op === 'mkdir') {
+          logger.debug(`Creating directory: ${call.params.path}`);
+          toolRes = await navigator.createDirectory(call.params.path!, call.params.mkdirOptions);
+        } else if (op === 'delete') {
+          logger.debug(`Deleting: ${call.params.path}`);
+          toolRes = await navigator.delete(call.params.path!, call.params.deleteOptions);
+        } else if (op === 'read') {
+          logger.debug(`Reading file: ${call.params.path}`);
+          toolRes = await navigator.readFile(call.params.path!);
+        } else if (op === 'list') {
+          logger.debug(`Listing directory: ${call.params.path ?? '.'}`);
+          toolRes = await navigator.listDirectory(call.params.path ?? '.');
+        } else if (op === 'stat') {
+          logger.debug(`Getting metadata: ${call.params.path}`);
+          toolRes = await navigator.getMetadata(call.params.path!);
+        } else {
+          logger.warn(`unknown navigate op: ${String(op)}`);
+          await traceEmit?.('toolcall.navigate.unknown_op', traceCtx, { op: String(op) });
+        }
+        
+        logger.info(`Navigate operation ${op} completed successfully`);
+      } catch (innerErr: any) {
+        logger.error(`navigate execution error: ${innerErr?.message ?? innerErr}`);
+        await traceEmit?.('toolcall.navigate.error', traceCtx, { error: innerErr?.message ?? String(innerErr) });
+      }
+
+      logger.debug(`Navigate result: ${JSON.stringify(toolRes ?? { ok: false })}`);
+      logger.info(`navigate result: ${JSON.stringify(toolRes ?? { ok: false })}`);
+      await traceEmit?.('toolcall.navigate.result', traceCtx, {
+        op,
+        ok: Boolean(toolRes?.ok),
+      });
+      return;
+    }
+
+    if (call.tool === 'versionControl') {
+      const allowed = Boolean(boxConfig.project?.versionControl?.before) || Boolean(boxConfig.project?.versionControl?.after);
+      if (!allowed) {
+        logger.warn(`versionControl requested but not authorized in config`);
+        await traceEmit?.('toolcall.versionControl.skipped', traceCtx, { reason: 'not_authorized' });
+        return;
+      }
+
+      try {
+        const params: any = { ...(call.params ?? {}) };
+        if (params.autoPush === undefined) params.autoPush = vcAutoPushConfig;
+        const vcRes = await attemptVersionControl(params);
+        logger.info(`versionControl result: ${JSON.stringify(vcRes ?? { ok: true })}`);
+        await traceEmit?.('toolcall.versionControl.result', traceCtx, { ok: true });
+      } catch (vcErr: any) {
+        logger.error(`versionControl error: ${vcErr?.message ?? vcErr}`);
+        await traceEmit?.('toolcall.versionControl.error', traceCtx, { error: vcErr?.message ?? String(vcErr) });
+      }
+      return;
+    }
+
+    logger.warn(`unhandled tool: ${String((call as any).tool)}`);
+  }
+}

package/core/loop/traceLogger.ts

@@ -0,0 +1,106 @@
+import path from 'node:path';
+import { mkdir, appendFile, readdir, stat, unlink } from 'node:fs/promises';
+import { STATES_LOGS_DIR } from '@core/paths/paths';
+import { Logger } from '@core/util/logger';
+
+export type TraceCtx = {
+  runId: string;
+  iter?: number;
+};
+
+export type TraceEvent = {
+  ts: string;
+  runId: string;
+  iter?: number;
+  event: string;
+  data?: Record<string, unknown>;
+};
+
+type TraceOptions = {
+  runId: string;
+  retain?: number;
+};
+
+function safeJsonStringify(v: unknown): string {
+  try {
+    return JSON.stringify(v);
+  } catch {
+    return JSON.stringify({ error: 'unstringifiable' });
+  }
+}
+
+function makeRunId(): string {
+  const rand = Math.random().toString(36).slice(2, 8);
+  return `${Date.now().toString(36)}-${rand}`;
+}
+
+async function applyRetention(retain: number): Promise<void> {
+  const entries = await readdir(STATES_LOGS_DIR, { withFileTypes: true }).catch(() => []);
+  const files = entries
+    .filter((e) => e.isFile() && e.name.startsWith('trace-') && e.name.endsWith('.jsonl'))
+    .map((e) => e.name);
+
+  const withTimes = await Promise.all(
+    files.map(async (name) => {
+      const p = path.join(STATES_LOGS_DIR, name);
+      const s = await stat(p).catch(() => null);
+      return { name, mtimeMs: s?.mtimeMs ?? 0 };
+    })
+  );
+
+  withTimes.sort((a, b) => b.mtimeMs - a.mtimeMs);
+  const toDelete = withTimes.slice(retain);
+  await Promise.all(toDelete.map((f) => unlink(path.join(STATES_LOGS_DIR, f.name)).catch(() => null)));
+}
+
+export function createTraceLogger(opts?: Partial<TraceOptions>) {
+  const runId = opts?.runId ?? process.env.BOXSAFE_RUN_ID ?? makeRunId();
+  const retain = typeof opts?.retain === 'number' ? opts.retain : Number(process.env.BOXSAFE_TRACE_RETAIN ?? 20);
+  const filePath = path.join(STATES_LOGS_DIR, `trace-${runId}.jsonl`);
+
+  let initialized = false;
+
+  async function ensureInit() {
+    if (initialized) return;
+    await mkdir(STATES_LOGS_DIR, { recursive: true });
+    if (Number.isFinite(retain) && retain > 0) {
+      await applyRetention(retain);
+    }
+    initialized = true;
+  }
+
+  async function emit(event: string, ctx?: TraceCtx, data?: Record<string, unknown>) {
+    await ensureInit();
+    const entry: TraceEvent = {
+      ts: new Date().toISOString(),
+      runId,
+      ...(typeof ctx?.iter === 'number' ? { iter: ctx.iter } : {}),
+      event,
+      ...(data ? { data } : {}),
+    };
+    await appendFile(filePath, `${safeJsonStringify(entry)}\n`, 'utf8');
+  }
+
+  function prefix(ctx?: TraceCtx): string {
+    const parts = [`run=${runId}`];
+    if (typeof ctx?.iter === 'number') parts.push(`iter=${ctx.iter}`);
+    return `[${parts.join('][')}]`;
+  }
+
+  function wrapLogger(ctx?: TraceCtx) {
+    const logger = Logger.createModuleLogger('Trace');
+    const p = prefix(ctx);
+    return {
+      info: (...a: any[]) => logger.info(`${p} ${a.join(' ')}`),
+      warn: (...a: any[]) => logger.warn(`${p} ${a.join(' ')}`),
+      error: (...a: any[]) => logger.error(`${p} ${a.join(' ')}`),
+    };
+  }
+
+  return {
+    runId,
+    filePath,
+    emit,
+    wrapLogger,
+  };
+}

package/core/loop/types.ts

@@ -0,0 +1,26 @@
+import type { LModel, LService } from '@ai/label';
+import type { CommandRun } from "../../types";
+import type { Navigator } from '@core/navigate';
+
+export interface LoopOptions {
+  service: LService;
+  model: LModel;
+  initialPrompt: string;
+  cmd: CommandRun;
+  lang: string;
+  pathOutput: string;
+  maxIterations?: number;
+  limit?: number;
+  signal?: AbortSignal;
+  pathGeneratedMarkdown?: string;
+  navigator?: Navigator;
+  workspace?: string;
+}
+
+export interface LoopResult {
+  ok: boolean;
+  iterations: number;
+  verdict?: unknown;
+  artifacts?: { outputFile?: string };
+  navigator?: Navigator | null;
+}

package/core/loop/versionControlAdapter.ts

@@ -0,0 +1,36 @@
+import { runVersionControl } from '@core/loop/git';
+import { Logger } from '@core/util/logger';
+
+const logger = Logger.createModuleLogger('VersionControlAdapter');
+
+type AttemptOptions = {
+  repoPath: string;
+  commitMessage: string;
+  autoPush: boolean;
+  generateNotes: boolean;
+};
+
+export function createVersionControlAttemptRunner() {
+  return async function attemptVersionControl(opts: AttemptOptions) {
+    const maxAttempts = 3;
+    let attempt = 0;
+    let lastErr: any = null;
+    logger.info(`attempting runVersionControl with opts=${JSON.stringify(opts)}`);
+    while (attempt < maxAttempts) {
+      attempt++;
+      try {
+        logger.debug(`attempt ${attempt}`);
+        const res = await runVersionControl(opts);
+        logger.info(`result (attempt ${attempt}): ${JSON.stringify(res)}`);
+        return res;
+      } catch (err: any) {
+        lastErr = err;
+        logger.warn(`attempt ${attempt} failed: ${err?.message ?? err}`);
+        const backoff = 300 * Math.pow(2, attempt - 1);
+        await new Promise((r) => setTimeout(r, backoff));
+      }
+    }
+    logger.error(`all ${maxAttempts} attempts failed: ${lastErr?.message ?? lastErr}`);
+    throw lastErr;
+  };
+}