llm-diff 0.1.0

package/LICENSE

@@ -0,0 +1,189 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work.
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to the Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by the Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding any notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   Copyright 2026 KeyGate Contributors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

package/README.md

@@ -0,0 +1,186 @@
+# llm-diff
+
+> `git diff` for prompts — compare LLM responses across prompt versions.
+
+See token count changes, cost deltas, latency shifts, and a word-level diff of the actual responses — all in one command.
+
+```
+$ llm-diff --a prompt-v1.txt --b prompt-v2.txt --model gpt-4o
+
+llm-diff  openai/gpt-4o
+
+  tokens          312 →        289   -23  (-7.4%)
+    input          45 →         38   -7
+    output        267 →        251   -16
+  cost       $0.0041 →    $0.0038   -$0.0003  (-7.3%)
+  latency      1247ms →      943ms   -304ms  (-24.4%)
+
+--- prompt A
++++ prompt B
+  The capital of France is Paris.
+- It is located in northern France and has a population of approximately 2.1 million people...
++ Paris, with ~2.1M residents, serves as the political and cultural center of the country...
+```
+
+## Install
+
+```bash
+npx llm-diff --a v1.txt --b v2.txt --model gpt-4o
+```
+
+Or install globally:
+
+```bash
+npm install -g llm-diff
+```
+
+## Quick start
+
+**1. Set your API key:**
+
+```bash
+export OPENAI_API_KEY=sk-...
+# or ANTHROPIC_API_KEY, GEMINI_API_KEY, GROQ_API_KEY
+```
+
+**2. Compare two prompts:**
+
+```bash
+# From files
+llm-diff --a prompt-v1.txt --b prompt-v2.txt --model gpt-4o
+
+# Inline text
+llm-diff -a "Explain gravity" -b "Explain gravity to a child" -m gpt-4o-mini
+
+# With a system prompt
+llm-diff -a v1.txt -b v2.txt -m claude-sonnet-4-20250514 -s "You are a science teacher"
+```
+
+## Usage
+
+```
+llm-diff --a <prompt-a> --b <prompt-b> --model <model> [options]
+```
+
+### Required
+
+| Flag | Description |
+|------|-------------|
+| `--a, -a` | Prompt A — file path or inline text |
+| `--b, -b` | Prompt B — file path or inline text |
+| `--model, -m` | Model name (see `--models` for full list) |
+
+### Options
+
+| Flag | Default | Description |
+|------|---------|-------------|
+| `--system, -s` | — | System prompt (file path or inline text) |
+| `--base-url` | — | Gateway URL override |
+| `--max-tokens` | 2048 | Max output tokens |
+| `--temperature` | 0 | Temperature |
+| `--timeout` | 60000 | Request timeout (ms) |
+| `--runs` | 1 | Number of runs to average |
+| `--no-parallel` | — | Run A and B sequentially |
+| `--full` | — | Show full inline diff with highlighting |
+| `--json` | — | JSON output for scripting |
+| `--models` | — | List supported models and pricing |
+
+## Supported models
+
+```bash
+llm-diff --models
+```
+
+### OpenAI
+`gpt-4o` · `gpt-4o-mini` · `gpt-4-turbo` · `gpt-4` · `gpt-3.5-turbo` · `o1` · `o1-mini` · `o3-mini`
+
+### Anthropic
+`claude-sonnet-4-20250514` · `claude-3.5-haiku` · `claude-3-opus`
+
+### Google Gemini
+`gemini-2.0-flash` · `gemini-2.0-pro` · `gemini-1.5-pro` · `gemini-1.5-flash`
+
+### Groq
+`llama-3.3-70b` · `llama-3.1-8b` · `mixtral-8x7b` · `gemma2-9b`
+
+## Gateway support
+
+Route requests through a custom gateway (like llmhut) instead of direct API calls:
+
+```bash
+llm-diff --a v1.txt --b v2.txt -m gpt-4o --base-url https://gw.llmhut.com/v1
+```
+
+The gateway handles authentication, so you don't need provider-specific API keys.
+
+## Averaging multiple runs
+
+LLM responses vary. Average over multiple runs for stable comparisons:
+
+```bash
+llm-diff --a v1.txt --b v2.txt -m gpt-4o --runs 5
+```
+
+Token counts and latency are averaged. The last response text is used for the diff.
+
+## JSON output
+
+Pipe results into scripts, dashboards, or eval pipelines:
+
+```bash
+llm-diff --a v1.txt --b v2.txt -m gpt-4o --json | jq '.delta'
+```
+
+```json
+{
+  "totalTokens": -23,
+  "totalTokensPct": -7.4,
+  "cost": -0.000293,
+  "costPct": -7.1,
+  "latencyMs": -304,
+  "latencyPct": -24.4
+}
+```
+
+## Programmatic API
+
+```javascript
+import { runDiff } from 'llm-diff';
+
+const result = await runDiff({
+  promptA: 'Explain gravity',
+  promptB: 'Explain gravity to a 5-year-old',
+  model: 'gpt-4o-mini',
+});
+
+console.log(result.delta);
+// { totalTokens: -23, cost: -0.0003, latencyMs: -304, ... }
+```
+
+## How it works
+
+1. Resolves the model → provider, pricing, API adapter
+2. Reads prompt A and B (from files or inline text)
+3. Fires both requests in parallel (or sequentially with `--no-parallel`)
+4. Collects token counts, cost, and latency from the API response
+5. Computes deltas between A and B
+6. Generates a word-level diff of the response text
+7. Renders everything to the terminal (or as JSON)
+
+## Roadmap
+
+- [ ] Eval pipeline integration (named experiments, history)
+- [ ] Side-by-side diff view
+- [ ] Cross-model comparison (`--model-a gpt-4o --model-b claude-sonnet-4-20250514`)
+- [ ] HTML report output
+- [ ] Config file support (`.llm-diff.json`)
+- [ ] Streaming output with live token counting
+- [ ] Mistral, Cohere, Together AI providers
+
+## Contributing
+
+See [CONTRIBUTING.md](CONTRIBUTING.md).
+
+## License
+
+Apache License — see [LICENSE](LICENSE).

package/bin/llm-diff.js

@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+
+import { run } from '../src/cli.js';
+
+run(process.argv.slice(2)).catch((err) => {
+  console.error(`\n\x1b[31m✖ ${err.message}\x1b[0m\n`);
+  process.exit(1);
+});

package/package.json

@@ -0,0 +1,56 @@
+{
+    "name": "llm-diff",
+    "version": "0.1.0",
+    "description": "Compare LLM responses across prompt versions — see token, cost, and latency deltas in a familiar diff format.",
+    "keywords": [
+      "llm",
+      "diff",
+      "prompt-engineering",
+      "openai",
+      "anthropic",
+      "gemini",
+      "groq",
+      "cli",
+      "developer-tools",
+      "ai",
+      "token-count",
+      "cost-comparison"
+    ],
+    "homepage": "https://github.com/llmhut/llm-diff",
+    "bugs": {
+      "url": "https://github.com/llmhut/llm-diff/issues"
+    },
+    "repository": {
+      "type": "git",
+      "url": "git+https://github.com/llmhut/llm-diff.git"
+    },
+    "license": "MIT",
+    "author": "llmhut",
+    "type": "module",
+    "main": "src/index.js",
+    "bin": {
+      "llm-diff": "bin/llm-diff.js"
+    },
+    "files": [
+      "bin/",
+      "src/",
+      "LICENSE",
+      "README.md"
+    ],
+    "scripts": {
+      "test": "vitest run",
+      "test:watch": "vitest",
+      "lint": "eslint src/ bin/ test/",
+      "llm-diff": "node bin/llm-diff.js"
+    },
+    "dependencies": {
+      "diff": "^7.0.0",
+      "tiktoken": "^1.0.18"
+    },
+    "devDependencies": {
+      "vitest": "^3.0.0"
+    },
+    "engines": {
+      "node": ">=18"
+    }
+  }

package/src/cli.js

@@ -0,0 +1,192 @@
+/**
+ * CLI entrypoint — parses args, validates, runs the diff, renders output.
+ *
+ * Zero dependencies for argument parsing (just process.argv).
+ */
+
+import { runDiff } from './diff.js';
+import { render, renderModelList } from './render.js';
+import { listModels } from './providers.js';
+
+const VERSION = '0.1.0';
+
+const HELP = `
+\x1b[1m\x1b[36mllm-diff\x1b[0m v${VERSION}
+Compare LLM responses across prompt versions.
+
+\x1b[1mUsage:\x1b[0m
+  llm-diff --a <prompt-a> --b <prompt-b> --model <model>
+
+\x1b[1mRequired:\x1b[0m
+  --a, -a <file|text>       Prompt A (file path or inline text)
+  --b, -b <file|text>       Prompt B (file path or inline text)
+  --model, -m <name>        Model to use (run --models to see list)
+
+\x1b[1mOptions:\x1b[0m
+  --system, -s <file|text>  System prompt (shared for both A and B)
+  --base-url <url>          Gateway URL override (e.g. llmhut gateway)
+  --max-tokens <n>          Max output tokens (default: 2048)
+  --temperature <n>         Temperature (default: 0)
+  --timeout <ms>            Request timeout (default: 60000)
+  --runs <n>                Number of runs to average (default: 1)
+  --no-parallel             Run A and B sequentially
+  --full                    Show full inline diff (not just changed lines)
+  --json                    Output as JSON (for scripting / eval pipelines)
+  --models                  List all supported models and pricing
+  --version, -v             Show version
+  --help, -h                Show this help
+
+\x1b[1mExamples:\x1b[0m
+  \x1b[2m# Compare two prompt files\x1b[0m
+  llm-diff --a prompt-v1.txt --b prompt-v2.txt --model gpt-4o
+
+  \x1b[2m# Inline prompts with system message\x1b[0m
+  llm-diff -a "Explain gravity simply" -b "Explain gravity to a 5-year-old" -m claude-sonnet-4-20250514 -s "You are a physics teacher"
+
+  \x1b[2m# Average over 3 runs, JSON output for CI\x1b[0m
+  llm-diff --a v1.txt --b v2.txt -m gpt-4o-mini --runs 3 --json
+
+  \x1b[2m# Route through llmhut gateway\x1b[0m
+  llm-diff --a v1.txt --b v2.txt -m gpt-4o --base-url https://gw.llmhut.com/v1
+
+\x1b[1mEnvironment variables:\x1b[0m
+  OPENAI_API_KEY            Required for OpenAI models
+  ANTHROPIC_API_KEY         Required for Anthropic models
+  GEMINI_API_KEY            Required for Gemini models
+  GROQ_API_KEY              Required for Groq models
+
+\x1b[2mhttps://github.com/llmhut/llm-diff\x1b[0m
+`;
+
+/**
+ * Parse argv into a flat options object.
+ */
+function parseArgs(argv) {
+  const opts = {};
+  const args = [...argv];
+
+  while (args.length) {
+    const arg = args.shift();
+
+    switch (arg) {
+      case '--a':
+      case '-a':
+        opts.promptA = args.shift();
+        break;
+      case '--b':
+      case '-b':
+        opts.promptB = args.shift();
+        break;
+      case '--model':
+      case '-m':
+        opts.model = args.shift();
+        break;
+      case '--system':
+      case '-s':
+        opts.system = args.shift();
+        break;
+      case '--base-url':
+        opts.baseUrl = args.shift();
+        break;
+      case '--max-tokens':
+        opts.maxTokens = parseInt(args.shift(), 10);
+        break;
+      case '--temperature':
+        opts.temperature = parseFloat(args.shift());
+        break;
+      case '--timeout':
+        opts.timeout = parseInt(args.shift(), 10);
+        break;
+      case '--runs':
+        opts.runs = parseInt(args.shift(), 10);
+        break;
+      case '--no-parallel':
+        opts.parallel = false;
+        break;
+      case '--full':
+        opts.full = true;
+        break;
+      case '--json':
+        opts.json = true;
+        break;
+      case '--models':
+        opts.listModels = true;
+        break;
+      case '--version':
+      case '-v':
+        opts.version = true;
+        break;
+      case '--help':
+      case '-h':
+        opts.help = true;
+        break;
+      default:
+        if (arg.startsWith('-')) {
+          throw new Error(`Unknown flag: ${arg}. Run llm-diff --help for usage.`);
+        }
+    }
+  }
+
+  return opts;
+}
+
+/**
+ * Main entry.
+ */
+export async function run(argv) {
+  const opts = parseArgs(argv);
+
+  if (opts.help) {
+    console.log(HELP);
+    return;
+  }
+
+  if (opts.version) {
+    console.log(VERSION);
+    return;
+  }
+
+  if (opts.listModels) {
+    renderModelList(listModels());
+    return;
+  }
+
+  // Validate required args
+  if (!opts.promptA) throw new Error('Missing --a (prompt A). Run llm-diff --help for usage.');
+  if (!opts.promptB) throw new Error('Missing --b (prompt B). Run llm-diff --help for usage.');
+  if (!opts.model)   throw new Error('Missing --model. Run llm-diff --help for usage.');
+
+  // Show spinner for non-JSON mode
+  let spinner;
+  if (!opts.json) {
+    spinner = startSpinner('Running prompts...');
+  }
+
+  try {
+    const result = await runDiff(opts);
+    if (spinner) spinner.stop();
+    render(result, { json: opts.json, full: opts.full });
+  } catch (err) {
+    if (spinner) spinner.stop();
+    throw err;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Minimal terminal spinner
+// ---------------------------------------------------------------------------
+
+function startSpinner(msg) {
+  const frames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
+  let i = 0;
+  const id = setInterval(() => {
+    process.stderr.write(`\r\x1b[36m${frames[i++ % frames.length]}\x1b[0m ${msg}`);
+  }, 80);
+
+  return {
+    stop() {
+      clearInterval(id);
+      process.stderr.write('\r\x1b[K'); // clear line
+    },
+  };
+}