@getollie/cli 0.0.1

package/src/cli.ts

@@ -0,0 +1,80 @@
+#!/usr/bin/env bun
+
+import { spawn } from "node:child_process";
+import process from "node:process";
+import { Command } from "@commander-js/extra-typings";
+import { buildPrompt } from "./prompt";
+import * as path from "node:path";
+
+// Manually parse arguments to handle -- separator
+const separatorIndex = process.argv.indexOf("--");
+const ollieArgs = separatorIndex === -1 ? process.argv : process.argv.slice(0, separatorIndex);
+const pochiArgs = separatorIndex === -1 ? [] : process.argv.slice(separatorIndex + 1);
+
+const program = new Command()
+  .name("ollie")
+  .description("Spawn a pochi subprocess with a web URL prompt");
+
+program
+  .command("eval", { isDefault: true })
+  .description("Evaluate a web URL")
+  .requiredOption("-u, --url <url>", "Web URL to evaluate")
+  .requiredOption("-q, --question <text>", "Original question/task that led to creating the sourceDir/url")
+  .requiredOption("-d, --dir <path>", "Source code directory to evaluate")
+  .action(async (options) => {
+    try {
+      const exitCode = await runPochi(options, pochiArgs);
+      process.exit(exitCode);
+    } catch (error) {
+      if (error instanceof Error) {
+        console.error(error.message);
+      } else {
+        console.error("Unknown error while running pochi");
+      }
+
+      process.exit(1);
+    }
+  });
+
+program
+  .command("library")
+  .description("Show the library directory")
+  .action(() => {
+    console.log(getChecklistDir());
+  });
+
+
+await program.parseAsync(ollieArgs);
+
+async function runPochi(options: {url: string, dir: string, question: string}, pochiArgs: string[]): Promise<number> {
+  const instructions = buildPrompt(options.url, options.dir, getChecklistDir(), options.question);
+  return new Promise((resolve, reject) => {
+    const child = spawn("pochi", pochiArgs, {
+      stdio: ["pipe", "inherit", "inherit"],
+      env: {
+        PATH: process.env.PATH,
+        POCHI_CUSTOM_INSTRUCTIONS: instructions,
+      }
+    });
+
+    child.on("error", (error) => {
+      reject(new Error(`Failed to start pochi: ${error.message}`));
+    });
+
+    child.stdin.write(`Please start evaluation\n`);
+    child.stdin.end();
+
+    child.on("close", (code, signal) => {
+      if (signal) {
+        reject(new Error(`pochi terminated with signal ${signal}`));
+        return;
+      }
+
+      resolve(code ?? 0);
+    });
+  });
+}
+
+function getChecklistDir(): string {
+  return path.join(__filename, "../../", "library");
+}

package/src/prompt.ts

@@ -0,0 +1,62 @@
+export function buildPrompt(url: string, sourceDir: string, checklistDir: string, question: string): string {
+  const questionContext = question ? `\nThe source directory and URL were created in response to the following task:\n"${question}"\n` : '';
+
+  const RuleTemplate = `
+You are a seasoned and meticulous code review expert, proficient in multiple programming languages, front-end technologies, and interaction design. Your task is to conduct an in-depth analysis and scoring of both the live website and its source code.
+
+The evaluation should cover implementation quality, design, architecture, performance, and adherence to best practices. Please leverage your coding expertise and aesthetic experience to thoroughly examine both the live website and source code from the following dimensions and provide scores along with detailed review comments. You should be very strict and cautious when giving full marks for each dimension.
+
+## Scoring criteria
+
+Before start evaluation, please read origin task carefully, and find 1-2 related example in ${checklistDir} and generate scoring criteria based on them. For each evaluation, there should be at most 10 criteria.
+
+
+## Role Definition
+
+**Responsibilities:** Act as an authoritative technical review committee member, ensuring objectivity, comprehensiveness, and impartiality.
+
+**Attitude:** Rigorous, professional, and unsparing, adept at identifying details and potential risks.
+
+**Additional Traits:** Possess exceptional aesthetic talent, with high standards for visual appeal and user experience.
+
+- The final output should be a JSON object containing the dimensions above, following this example:
+
+\`\`\`json
+{
+  "score": 32,
+  "checklist": [
+    {
+      "title": "Is the security checklist (SCL) content fully implemented in the webpage?",
+      "reasoning": "The SCL content is fully implemented in the webpage, with all items listed and accessible.",
+      "score": 8
+    },
+    ...
+  ]
+}
+\`\`\`
+
+## Question Context
+
+${questionContext}
+
+Please evaluate the following website and source code according to the standards above:
+
+**URL to evaluate:** ${url}
+
+**Source code location:** ${sourceDir} - read ONLY page.tsx file(s)
+
+## Instructions:
+1. Visit the URL and thoroughly analyze the live website
+2. Take snapshot if needed to capture the visual design and user experience
+3. Read ONLY the page.tsx file(s) in the directory: ${sourceDir} to understand the implementation
+4. Analyze code structure and organization based on the page.tsx file
+5. Check for proper error handling, testing, and documentation in the page.tsx file
+6. Evaluate based on all the scoring criteria provided
+7. Provide detailed reasoning for your scores, citing specific examples from both the live site and source code
+8. Output the final score as a JSON object followed by your detailed analysis
+
+Begin your evaluation now.
+`.trim();
+
+  return RuleTemplate;
+}