assertloom 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +393 -2
- package/dist/ai/anthropic.d.ts +11 -0
- package/dist/ai/anthropic.js +68 -0
- package/dist/ai/anthropic.js.map +1 -0
- package/dist/ai/index.d.ts +12 -0
- package/dist/ai/index.js +23 -0
- package/dist/ai/index.js.map +1 -0
- package/dist/ai/openai.d.ts +9 -0
- package/dist/ai/openai.js +57 -0
- package/dist/ai/openai.js.map +1 -0
- package/dist/ai/openaiCompatible.d.ts +13 -0
- package/dist/ai/openaiCompatible.js +62 -0
- package/dist/ai/openaiCompatible.js.map +1 -0
- package/dist/ai/pricing.d.ts +38 -0
- package/dist/ai/pricing.js +105 -0
- package/dist/ai/pricing.js.map +1 -0
- package/dist/ai/prompts.d.ts +27 -0
- package/dist/ai/prompts.js +89 -0
- package/dist/ai/prompts.js.map +1 -0
- package/dist/ai/provider.d.ts +26 -0
- package/dist/ai/provider.js +4 -0
- package/dist/ai/provider.js.map +1 -0
- package/dist/ai/redact.d.ts +5 -0
- package/dist/ai/redact.js +31 -0
- package/dist/ai/redact.js.map +1 -0
- package/dist/analyzer/dependencies.d.ts +16 -0
- package/dist/analyzer/dependencies.js +168 -0
- package/dist/analyzer/dependencies.js.map +1 -0
- package/dist/analyzer/format.d.ts +3 -0
- package/dist/analyzer/format.js +14 -0
- package/dist/analyzer/format.js.map +1 -0
- package/dist/analyzer/functions.d.ts +4 -0
- package/dist/analyzer/functions.js +324 -0
- package/dist/analyzer/functions.js.map +1 -0
- package/dist/analyzer/index.d.ts +5 -0
- package/dist/analyzer/index.js +29 -0
- package/dist/analyzer/index.js.map +1 -0
- package/dist/analyzer/parser.d.ts +11 -0
- package/dist/analyzer/parser.js +38 -0
- package/dist/analyzer/parser.js.map +1 -0
- package/dist/analyzer/usage.d.ts +16 -0
- package/dist/analyzer/usage.js +159 -0
- package/dist/analyzer/usage.js.map +1 -0
- package/dist/ci/report.d.ts +88 -0
- package/dist/ci/report.js +139 -0
- package/dist/ci/report.js.map +1 -0
- package/dist/ci/run.d.ts +70 -0
- package/dist/ci/run.js +198 -0
- package/dist/ci/run.js.map +1 -0
- package/dist/ci/select.d.ts +58 -0
- package/dist/ci/select.js +238 -0
- package/dist/ci/select.js.map +1 -0
- package/dist/cli/commands/analyze.d.ts +2 -0
- package/dist/cli/commands/analyze.js +49 -0
- package/dist/cli/commands/analyze.js.map +1 -0
- package/dist/cli/commands/ci.d.ts +37 -0
- package/dist/cli/commands/ci.js +258 -0
- package/dist/cli/commands/ci.js.map +1 -0
- package/dist/cli/commands/generate.d.ts +3 -0
- package/dist/cli/commands/generate.js +201 -0
- package/dist/cli/commands/generate.js.map +1 -0
- package/dist/cli/commands/run.d.ts +4 -0
- package/dist/cli/commands/run.js +53 -0
- package/dist/cli/commands/run.js.map +1 -0
- package/dist/cli/index.d.ts +2 -0
- package/dist/cli/index.js +16 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/program.d.ts +2 -0
- package/dist/cli/program.js +20 -0
- package/dist/cli/program.js.map +1 -0
- package/dist/cli/report.d.ts +3 -0
- package/dist/cli/report.js +36 -0
- package/dist/cli/report.js.map +1 -0
- package/dist/cli/validateFile.d.ts +5 -0
- package/dist/cli/validateFile.js +43 -0
- package/dist/cli/validateFile.js.map +1 -0
- package/dist/errors.d.ts +4 -0
- package/dist/errors.js +8 -0
- package/dist/errors.js.map +1 -0
- package/dist/generator/diff.d.ts +5 -0
- package/dist/generator/diff.js +56 -0
- package/dist/generator/diff.js.map +1 -0
- package/dist/generator/importPlan.d.ts +15 -0
- package/dist/generator/importPlan.js +62 -0
- package/dist/generator/importPlan.js.map +1 -0
- package/dist/generator/index.d.ts +5 -0
- package/dist/generator/index.js +6 -0
- package/dist/generator/index.js.map +1 -0
- package/dist/generator/mocks.d.ts +48 -0
- package/dist/generator/mocks.js +356 -0
- package/dist/generator/mocks.js.map +1 -0
- package/dist/generator/schema.d.ts +40 -0
- package/dist/generator/schema.js +15 -0
- package/dist/generator/schema.js.map +1 -0
- package/dist/generator/testGenerator.d.ts +36 -0
- package/dist/generator/testGenerator.js +85 -0
- package/dist/generator/testGenerator.js.map +1 -0
- package/dist/generator/testWriter.d.ts +22 -0
- package/dist/generator/testWriter.js +93 -0
- package/dist/generator/testWriter.js.map +1 -0
- package/dist/generator/validate.d.ts +31 -0
- package/dist/generator/validate.js +180 -0
- package/dist/generator/validate.js.map +1 -0
- package/dist/repair/classifyFailure.d.ts +11 -0
- package/dist/repair/classifyFailure.js +26 -0
- package/dist/repair/classifyFailure.js.map +1 -0
- package/dist/repair/index.d.ts +2 -0
- package/dist/repair/index.js +3 -0
- package/dist/repair/index.js.map +1 -0
- package/dist/repair/repairLoop.d.ts +74 -0
- package/dist/repair/repairLoop.js +219 -0
- package/dist/repair/repairLoop.js.map +1 -0
- package/dist/repair/schema.d.ts +17 -0
- package/dist/repair/schema.js +14 -0
- package/dist/repair/schema.js.map +1 -0
- package/dist/runner/coverage.d.ts +5 -0
- package/dist/runner/coverage.js +42 -0
- package/dist/runner/coverage.js.map +1 -0
- package/dist/runner/index.d.ts +5 -0
- package/dist/runner/index.js +5 -0
- package/dist/runner/index.js.map +1 -0
- package/dist/runner/jestConfig.d.ts +11 -0
- package/dist/runner/jestConfig.js +103 -0
- package/dist/runner/jestConfig.js.map +1 -0
- package/dist/runner/jestRunner.d.ts +17 -0
- package/dist/runner/jestRunner.js +154 -0
- package/dist/runner/jestRunner.js.map +1 -0
- package/dist/runner/networkGuard.d.ts +4 -0
- package/dist/runner/networkGuard.js +55 -0
- package/dist/runner/networkGuard.js.map +1 -0
- package/dist/runner/project.d.ts +30 -0
- package/dist/runner/project.js +86 -0
- package/dist/runner/project.js.map +1 -0
- package/dist/runner/results.d.ts +32 -0
- package/dist/runner/results.js +58 -0
- package/dist/runner/results.js.map +1 -0
- package/dist/runner/types.d.ts +39 -0
- package/dist/runner/types.js +2 -0
- package/dist/runner/types.js.map +1 -0
- package/dist/runner/vitest.d.ts +24 -0
- package/dist/runner/vitest.js +105 -0
- package/dist/runner/vitest.js.map +1 -0
- package/dist/types/index.d.ts +96 -0
- package/dist/types/index.js +2 -0
- package/dist/types/index.js.map +1 -0
- package/package.json +67 -4
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ish0212
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,394 @@
|
|
|
1
|
-
#
|
|
1
|
+
# Assertloom
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
[](https://www.npmjs.com/package/assertloom)
|
|
4
|
+
[](https://github.com/Ish0212/Assertloom/actions/workflows/ci.yml)
|
|
5
|
+
[](https://www.npmjs.com/package/assertloom#provenance)
|
|
6
|
+
[](package.json)
|
|
7
|
+
[](LICENSE)
|
|
8
|
+
|
|
9
|
+
Assertloom is an open-source CLI that reads a JavaScript or TypeScript file, generates Jest unit tests for it, runs them, repairs the ones that fail, and reports coverage. It doesn't assume generated tests are correct. Every test is run before it is kept.
|
|
10
|
+
|
|
11
|
+

|
|
12
|
+
|
|
13
|
+
<sub>A real session on [`examples/sample-project`](examples/sample-project/README.md), recorded with `scripts/record-demo.mjs` (pauses for the model shortened to 2 seconds). The raw recording is in `docs/demo.cast` and plays with `asciinema play docs/demo.cast`.</sub>
|
|
14
|
+
|
|
15
|
+
> **Status: 0.2.0.** `analyze` statically extracts functions, dependencies, and how each import is used. `generate` writes Jest tests for every exported function, mocks the dependencies that do I/O (packages, databases, the file system, `fetch`), runs the tests with network access blocked, and repairs failing tests in up to three rounds within a spending limit. `run` reports pass/fail counts and coverage for any test file. `ci` and the [GitHub Action](#github-action) do all of this for the files a pull request changes. On 9 files from 9 open-source packages, all 73 kept tests pass and mean line coverage goes from 0% to 88.6%, for $0.07 in total ([benchmark](#benchmark)).
|
|
16
|
+
|
|
17
|
+
## Install
|
|
18
|
+
|
|
19
|
+
Assertloom needs Node.js 22 or newer and, for `generate`, an [OpenAI API key](https://platform.openai.com/api-keys).
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
npm install --save-dev assertloom
|
|
23
|
+
export OPENAI_API_KEY=sk-... # PowerShell: $env:OPENAI_API_KEY = "sk-..."
|
|
24
|
+
|
|
25
|
+
npx assertloom generate src/utils.ts
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
You can also run it without installing: `npx assertloom@latest generate src/utils.ts`. Your project doesn't need Jest; Assertloom brings its own and uses your Jest and config if you have them.
|
|
29
|
+
|
|
30
|
+
To see it work on a prepared project first, follow [`examples/sample-project`](examples/sample-project/README.md). It walks through pure logic, an HTTP client with `fetch`, and file system code.
|
|
31
|
+
|
|
32
|
+
## Run from source
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
git clone https://github.com/Ish0212/Assertloom.git
|
|
36
|
+
cd Assertloom
|
|
37
|
+
npm install
|
|
38
|
+
|
|
39
|
+
npm run dev -- analyze examples/math.ts
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Example output:
|
|
43
|
+
|
|
44
|
+
```
|
|
45
|
+
✓ Parsed examples/math.ts (TypeScript, ESM)
|
|
46
|
+
✓ Found 2 exported functions
|
|
47
|
+
✓ Found 0 dependencies (0 external, 0 local, 0 built-in)
|
|
48
|
+
|
|
49
|
+
add(a: number, b: number): number
|
|
50
|
+
divide(a: number, b: number): number [throws]
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
Add `--json` for the full analysis object. Each function includes its export names, kind (declaration, arrow, or expression), async/generator flags, parameters (name, type, optional, rest, default value), return type, type parameters, whether it throws directly, and the imported modules it uses.
|
|
54
|
+
|
|
55
|
+
## What the analyzer understands
|
|
56
|
+
|
|
57
|
+
- ESM and CommonJS: `export function`, `export const fn = () => {}`, `export { a as b }`, `export default`, `module.exports = { ... }`, `module.exports = fn`, `exports.x = fn`, and an object declared first and exported later (`const api = { parse }; module.exports = api`, `exports.parse = api.parse`)
|
|
58
|
+
- TypeScript: parameter and return types, generics, overloads (only the implementation is kept), `declare` (skipped), and `satisfies`/`as` wrappers
|
|
59
|
+
- Imports: `import`, type-only imports (excluded from runtime dependencies), re-exports, `require()`, and dynamic `import()`, each classified as local, external package, or Node built-in
|
|
60
|
+
- Per-function dependencies, including top-level values built from imports (for example `const prisma = new PrismaClient()`), while ignoring parameters that shadow an import
|
|
61
|
+
- How each imported value is used: called, constructed with `new`, used with `instanceof`, which members are accessed, and whether a call's result is kept (and whether that happens when the module loads). Mocks are built from this.
|
|
62
|
+
- Uses of the global `fetch` (unless the code declares its own `fetch`)
|
|
63
|
+
|
|
64
|
+
UMD wrappers (`(function (root, factory) { ... })(this, function () { ... })`) are recognized: whatever the factory returns is treated as the module's exports. Classes and object methods outside `module.exports` are not analyzed yet.
|
|
65
|
+
|
|
66
|
+
## Generating tests
|
|
67
|
+
|
|
68
|
+
By default `generate` uses OpenAI and needs an API key in the `OPENAI_API_KEY` environment variable. Anthropic and local or OpenAI-compatible models are supported too; see below.
|
|
69
|
+
|
|
70
|
+
```powershell
|
|
71
|
+
# PowerShell (current session)
|
|
72
|
+
$env:OPENAI_API_KEY = "sk-..."
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
# bash/zsh
|
|
77
|
+
export OPENAI_API_KEY=sk-...
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
npx assertloom generate examples/strings.ts
|
|
82
|
+
npx assertloom generate examples/math.ts --dry-run # print instead of writing
|
|
83
|
+
npx assertloom generate examples/math.ts --model gpt-5.4 # pick a model
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
From a source checkout, use `npm run dev -- generate ...` instead of `npx assertloom generate ...`.
|
|
87
|
+
|
|
88
|
+
The default model is `gpt-5.4-mini`. Override it with `--model` or `ASSERTLOOM_MODEL`.
|
|
89
|
+
|
|
90
|
+
Other providers are selected with `--provider`:
|
|
91
|
+
|
|
92
|
+
| Provider | Key | Default model | Notes |
|
|
93
|
+
| --- | --- | --- | --- |
|
|
94
|
+
| `openai` (default) | `OPENAI_API_KEY` | `gpt-5.4-mini` | |
|
|
95
|
+
| `anthropic` | `ANTHROPIC_API_KEY` | `claude-haiku-5-5` | Uses structured JSON output |
|
|
96
|
+
| `openai-compatible` | `ASSERTLOOM_API_KEY` (optional) | none; `--model` is required | Any server with the OpenAI Chat Completions API and JSON-schema output: Ollama, LM Studio, vLLM, OpenRouter. Set the URL with `--base-url` or `ASSERTLOOM_BASE_URL`. |
|
|
97
|
+
|
|
98
|
+
```bash
|
|
99
|
+
npx assertloom generate src/price.ts --provider anthropic --model claude-sonnet-5-5
|
|
100
|
+
npx assertloom generate src/price.ts --provider openai-compatible --base-url http://localhost:11434/v1 --model qwen3-coder
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
The spending limit uses published prices for OpenAI and Anthropic models. For models without a known price (local or OpenAI-compatible ones), `--max-cost` can't be applied, so each run is capped at 500,000 tokens instead.
|
|
104
|
+
|
|
105
|
+
`examples/strings.test.ts`, `examples/math.test.ts`, and `examples/users.test.ts` are unedited output from `gpt-5.4-mini`. For `strings.ts`, 24 of 26 generated tests passed on the first run, and one repair round fixed the other 2, ending at 26/26 passing with 100% coverage. `users.ts` depends on two external packages (`axios` and `pg`); see [Mocking dependencies](#mocking-dependencies).
|
|
106
|
+
|
|
107
|
+
How it works:
|
|
108
|
+
|
|
109
|
+
1. The analyzer finds the exported functions and every dependency they use.
|
|
110
|
+
2. Assertloom decides which dependencies to mock and writes the mocks itself (see below).
|
|
111
|
+
3. The model receives the source file, the function signatures, the exact names to call them by, and a description of the mocks. It returns structured test cases (name, category, body) validated with Zod, not a whole file.
|
|
112
|
+
4. Each case is checked before it is kept: it must parse, call `expect()`, and must not use `import`, `require`, `jest.mock`, `jest.requireActual` and similar module-loading calls, `process` methods that affect the test run (`exit`, `chdir`, ...), `module.exports`, or its own `describe`/`it`. It also must not stub or spy on the functions under test: calls inside the source file bypass spies, so only dependencies can be controlled. Rejected cases are listed with the reason.
|
|
113
|
+
5. Assertloom writes the mocks, imports, and `describe`/`it` blocks itself, parses the finished file, and saves it as `<name>.test.preview.<ext>` next to the source. Jest's default pattern ignores that name, because the tests have not been run yet.
|
|
114
|
+
6. Assertloom runs the preview with Jest. If every test passes, it is saved as `<name>.test.<ext>` and the preview is deleted. If any fail, Assertloom tries to repair them (see below), and keeps the preview for review if failures remain. Use `--no-run` to skip running and repairing.
|
|
115
|
+
|
|
116
|
+
If the destination test file already exists, Assertloom prints a unified diff of the changes before overwriting it. Files it did not generate are never overwritten without `--force`.
|
|
117
|
+
|
|
118
|
+
Each command prints its token usage and estimated cost (`Tokens: 2,431 in / 1,020 out, ≈$0.0064`), and `--report` includes the cost. Before every model call, Assertloom checks the worst case (the full prompt plus the maximum output) against `--max-cost` (default $1) and stops instead of going over. Models without a known price are capped at 500,000 tokens.
|
|
119
|
+
|
|
120
|
+
## Mocking dependencies
|
|
121
|
+
|
|
122
|
+
Assertloom writes every `jest.mock()` call itself, based on how the source uses each import. The model never writes mocks; it only sets return values inside each test (`mockResolvedValue`, `mockRejectedValue`, and so on). Mocked modules are never loaded, so tests don't touch real databases, APIs, or files, and the packages don't even need to be installed.
|
|
123
|
+
|
|
124
|
+
`examples/users.ts` loads users from Postgres through `pg` and falls back to an HTTP API through `axios`:
|
|
125
|
+
|
|
126
|
+
```
|
|
127
|
+
✓ Analyzed 3 exported functions (3 tested)
|
|
128
|
+
✓ Mocked 2 dependencies: axios, pg
|
|
129
|
+
- axios: external package, not installed (virtual mock)
|
|
130
|
+
- pg: external package, not installed (virtual mock)
|
|
131
|
+
✓ Generated 9 test cases (5 normal, 2 edge, 2 error)
|
|
132
|
+
✓ Ran Jest: 9 passed, 0 failed (bundled Jest, Assertloom default config, network blocked, 2.9s)
|
|
133
|
+
✓ Coverage: 100% lines, 100% branches, 100% functions
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
The generated file starts with mocks shaped like the code that uses them:
|
|
137
|
+
|
|
138
|
+
```ts
|
|
139
|
+
jest.mock("axios", () => ({ __esModule: true, default: { get: jest.fn(), isAxiosError: jest.fn() } }), { virtual: true });
|
|
140
|
+
jest.mock("pg", () => {
|
|
141
|
+
const instance1 = { query: jest.fn() };
|
|
142
|
+
return { __esModule: true, Pool: function () { return instance1; } };
|
|
143
|
+
}, { virtual: true });
|
|
144
|
+
|
|
145
|
+
import axios from "axios";
|
|
146
|
+
import { Pool } from "pg";
|
|
147
|
+
import { getUser, importUser, countUsersByDomain } from "./users";
|
|
148
|
+
|
|
149
|
+
const pool = new Pool();
|
|
150
|
+
|
|
151
|
+
beforeEach(() => {
|
|
152
|
+
jest.resetAllMocks();
|
|
153
|
+
});
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
What gets mocked:
|
|
157
|
+
|
|
158
|
+
| Dependency | Default |
|
|
159
|
+
| --- | --- |
|
|
160
|
+
| External packages | Mocked, except utility libraries without I/O (`zod`, `lodash`, `date-fns`, and similar), which stay real |
|
|
161
|
+
| Node built-ins | `fs`, `child_process`, `http`, and other built-ins with side effects are mocked; `path`, `url`, `util`, and similar stay real |
|
|
162
|
+
| Local modules | Assertloom reads them (without running them). A module is mocked if it, or anything it imports, does I/O; otherwise the real code runs |
|
|
163
|
+
| Global `fetch` | Replaced with a `jest.fn()` for the test file and restored afterwards |
|
|
164
|
+
|
|
165
|
+
Override the defaults with `--mock <module>` and `--keep-real <module>` (both repeatable), using the specifier as written in the source or the package name.
|
|
166
|
+
|
|
167
|
+
How mocks are shaped:
|
|
168
|
+
|
|
169
|
+
- Functions become `jest.fn()`; objects with methods become objects of `jest.fn()`s; default, named, namespace, and CommonJS imports each get the matching module shape.
|
|
170
|
+
- Classes, and objects created when the module loads (`const pool = new Pool()`, `const api = axios.create()`), return one shared mock object. The test file defines a matching variable (`const pool = new Pool()`), so tests configure the same object the source uses.
|
|
171
|
+
- Every mock is reset before each test, so each test sets up exactly what it needs, including rejection paths (`mockRejectedValue`) for error handling.
|
|
172
|
+
- Packages that are not installed get `{ virtual: true }` mocks, and local paths are rewritten relative to the test file.
|
|
173
|
+
- Pure built-ins the source imports (such as `path`) are imported into the test too, so expected values like file paths are built the same way and pass on every operating system.
|
|
174
|
+
|
|
175
|
+
`fixtures/mocking/` covers the main dependency types, each run through real Jest by Assertloom's test suite:
|
|
176
|
+
|
|
177
|
+
| Fixture | Dependency | Module system |
|
|
178
|
+
| --- | --- | --- |
|
|
179
|
+
| `examples/users.ts` | `axios` + `pg` pool created at load | TypeScript ESM |
|
|
180
|
+
| `weather.ts` | global `fetch` | TypeScript ESM |
|
|
181
|
+
| `orders.cjs` | local `./db.cjs` wrapping `pg` (mocked); `./money.cjs` (kept real) | CommonJS |
|
|
182
|
+
| `settings.mjs` | `node:fs/promises` (mocked); `node:path` (kept real) | `.mjs` |
|
|
183
|
+
| `esm-project/git.js` | `node:child_process` | `.js` in a `"type": "module"` package |
|
|
184
|
+
|
|
185
|
+
All five also passed against `gpt-5.4-mini` with 100% line coverage.
|
|
186
|
+
|
|
187
|
+
Current limitations:
|
|
188
|
+
|
|
189
|
+
- Modules loaded with dynamic `import()` are not mocked.
|
|
190
|
+
- Projects whose Jest config runs native ESM (`transform: {}` with `--experimental-vm-modules`) don't apply `jest.mock()`; the bundled config is not affected.
|
|
191
|
+
- Generated TypeScript tests run correctly, but may not pass `tsc` type checking when the model calls mock methods without `jest.mocked(...)`.
|
|
192
|
+
- Classes exported by the source are not tested yet.
|
|
193
|
+
|
|
194
|
+
## Repairing failing tests
|
|
195
|
+
|
|
196
|
+
When generated tests fail, `generate` sends the model the failing test bodies, a trimmed failure message for each, and the source file marked read-only. It then reruns Jest. For each failing test, the model chooses one of three actions:
|
|
197
|
+
|
|
198
|
+
- **fix**: replace the test body, for example to correct a wrong expected value or a missing `await`. The new body goes through the same checks as a generated one.
|
|
199
|
+
- **skip**: the test looks right and the source looks wrong. The test is kept as `it.skip` with a comment explaining the suspected bug, so a real bug is not hidden by changing the assertion.
|
|
200
|
+
- **remove**: the test cannot be made reliable.
|
|
201
|
+
|
|
202
|
+
Failures are classified before anything is sent. Assertion, runtime, and timeout failures can be repaired, including tests that configured a mock wrongly or not at all. Failures caused by a real network connection (a dependency that was not mocked, for example after `--keep-real`) are set aside, and failures that stop the whole file from running (missing modules, syntax or config problems) end the loop, because editing test bodies cannot fix them.
|
|
203
|
+
|
|
204
|
+
Limits:
|
|
205
|
+
|
|
206
|
+
- At most 3 rounds (`--max-attempts 0` to `3`; `--no-repair` turns repair off). The loop also stops when everything passes, when the model proposes no usable fix, or before a round that could exceed `--max-cost`.
|
|
207
|
+
- A round that makes a previously passing test fail is undone, and the loop stops.
|
|
208
|
+
- Tests still failing at the end are kept as `it.skip` with the reason, so the file runs cleanly and the problems stay visible.
|
|
209
|
+
- Repair only writes the generated test file. Every Jest run snapshots the source file first and restores it afterwards if a test changed it.
|
|
210
|
+
|
|
211
|
+
Use `--report <file>` to save JSON with the results of the first run, every repair round (requested changes, rejected proposals, before/after counts, token usage), and the final run.
|
|
212
|
+
|
|
213
|
+
`fixtures/repair/` holds a documented set of intentionally failing tests (wrong values, a misspelled function name, a missing `await`, a real bug in the source, and a `fetch` mock the test never configured), with the repairs a model would propose. Assertloom's test suite runs that scenario through real Jest.
|
|
214
|
+
|
|
215
|
+
## Running tests
|
|
216
|
+
|
|
217
|
+
`run` executes any test file and reports results and coverage for its source file, which is inferred from the name (`math.test.preview.ts` → `math.ts`) or passed with `--source`.
|
|
218
|
+
|
|
219
|
+
```bash
|
|
220
|
+
npx assertloom run fixtures/runner/failing.test.preview.ts --source fixtures/runner/calc.ts
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
```
|
|
224
|
+
! Ran Jest: 1 passed, 3 failed (bundled Jest, Assertloom default config, network blocked, 9.3s)
|
|
225
|
+
✗ add › has a wrong expectation [assertion]
|
|
226
|
+
Error: expect(received).toBe(expected) // Object.is equality
|
|
227
|
+
Expected: 5
|
|
228
|
+
Received: 4
|
|
229
|
+
✗ divide › calls a method that does not exist [error]
|
|
230
|
+
TypeError: result.toFixedd is not a function
|
|
231
|
+
✗ sign › waits forever [timeout]
|
|
232
|
+
Error: thrown: "Exceeded timeout of 5000 ms for a test.
|
|
233
|
+
✓ Coverage: 71.4% lines, 66.7% branches, 66.7% functions
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
Add `--json` for the full result and `--timeout <seconds>` to change the 120-second limit.
|
|
237
|
+
|
|
238
|
+
How the runner works:
|
|
239
|
+
|
|
240
|
+
- **Jest setup.** Assertloom finds the project root (the nearest `package.json`, without leaving the git repository). If the project has its own Jest config (`jest.config.js/.cjs/.mjs/.json/.ts/.mts/.cts` or a `"jest"` key in `package.json`), Assertloom uses the project's Jest with that config and overrides only test selection and coverage settings. TypeScript configs are loaded with Node's built-in type stripping (Node.js 22.18 or newer; older versions fall back to the default config). Otherwise it uses its bundled Jest with Babel for TypeScript and JSX, so projects without Jest work too. Multi-project configs fall back to the default config for now.
|
|
241
|
+
- **Module systems.** The bundled config compiles ESM to CommonJS for Jest, so `.mjs` files, `.js` files in `"type": "module"` packages, `.cjs`, and TypeScript all run without `--experimental-vm-modules`, and `jest.mock()` works the same way in all of them.
|
|
242
|
+
- **Isolation.** Jest runs in a child process, one file at a time (`--runInBand`), with a hard timeout and a 5-second per-test timeout. Environment variables that look like credentials (`*KEY*`, `*TOKEN*`, `*SECRET*`, `*PASSWORD*`, and similar) are removed before the tests run.
|
|
243
|
+
- **Network.** Network access is blocked by default. A preloaded guard (`--require`, inherited by any Node subprocess the tests start) fails every outgoing TCP connection and DNS lookup with `Assertloom blocked a network connection`, which covers `net`, `http`, `https`, `tls`, and `fetch`. Tests can still start local servers, but cannot connect to them. Pass `--allow-network` to `run` or `generate` to turn the guard off. The guard stops accidental connections from well-behaved code; it is not a sandbox against hostile code (which could, for example, spawn a non-Node process).
|
|
244
|
+
- **Results.** Each test is reported as passed, failed, or skipped. Failures are labeled `assertion` (an `expect` did not match), `error` (an exception), or `timeout`. Problems that stop the whole file from running are reported separately as infrastructure failures: `module-not-found`, `transform` (syntax or module-format problems), `suite-error`, `crash`, or `timeout`.
|
|
245
|
+
- **Coverage.** Measured with V8 for the source file only, as line, statement, function, and branch percentages.
|
|
246
|
+
|
|
247
|
+
## GitHub Action
|
|
248
|
+
|
|
249
|
+
Assertloom also runs as a GitHub Action on pull requests. It generates, runs, and repairs tests for the changed JavaScript and TypeScript files under one shared spending limit, writes a job summary, and uploads the generated tests and a JSON report as an artifact. It never commits, pushes, or comments.
|
|
250
|
+
|
|
251
|
+
```yaml
|
|
252
|
+
- uses: actions/checkout@v5
|
|
253
|
+
with:
|
|
254
|
+
fetch-depth: 0
|
|
255
|
+
persist-credentials: false
|
|
256
|
+
- uses: actions/setup-node@v5
|
|
257
|
+
with:
|
|
258
|
+
node-version: 22
|
|
259
|
+
- run: npm ci
|
|
260
|
+
- uses: Ish0212/Assertloom@v0.2.0
|
|
261
|
+
with:
|
|
262
|
+
api-key: ${{ secrets.OPENAI_API_KEY }}
|
|
263
|
+
max-cost: "0.50"
|
|
264
|
+
```
|
|
265
|
+
|
|
266
|
+
See [docs/github-action.md](docs/github-action.md) for the complete workflow (including the fork guard), inputs and outputs, budget and failure behavior, and the trust model. A copyable workflow is in [`examples/github-action/assertloom.yml`](examples/github-action/assertloom.yml). The same pipeline runs locally with `assertloom ci --base origin/main`.
|
|
267
|
+
|
|
268
|
+
## Benchmark
|
|
269
|
+
|
|
270
|
+
`npm run bench` downloads one file from each of nine independent open-source packages, pinned to exact versions (`bench/targets.ts`), and runs the full pipeline on each: generate, run, and repair. It records generation, execution, and repair success rates, runtime, model cost, and coverage. Results are written to `bench/results/latest.json` and `latest.md`, and the final test files to `bench/results/tests/`. It needs `OPENAI_API_KEY`; use `--only ms,cookie` for a subset and `--model` to compare models.
|
|
271
|
+
|
|
272
|
+
In summary: Assertloom generated runnable test suites for all nine files, repaired all nine tests that failed on the first run, and reached 88.6% mean line coverage.
|
|
273
|
+
|
|
274
|
+
Results from 2026-11-01 with `gpt-5.4-mini`, Node 24.11.1 on Windows, $0.50 limit per file:
|
|
275
|
+
|
|
276
|
+
| Target | Functions tested | Valid cases | First run | Final | Repair rounds | Lines | Branches | Time | Cost | Mocked |
|
|
277
|
+
| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
|
|
278
|
+
| `ms@2.1.3` index.js | 1/1 | 6/6 | 4/6 | 6/6 passing | 1 | 88.88% | 50.81% | 18.0s | $0.0056 | – |
|
|
279
|
+
| `bytes@3.1.2` index.js | 3/3 | 15/15 | 12/15 | 15/15 passing | 2 | 97.64% | 90.9% | 16.5s | $0.0093 | – |
|
|
280
|
+
| `escape-html@1.0.3` index.js | 1/1 | 5/5 | 5/5 | 5/5 passing | 0 | 100% | 100% | 6.1s | $0.0026 | – |
|
|
281
|
+
| `content-type@1.0.5` index.js | 2/2 | 9/10 | 9/9 | 9/9 passing | 0 | 96.44% | 88.88% | 7.3s | $0.0053 | – |
|
|
282
|
+
| `cookie@0.7.2` index.js | 2/2 | 10/10 | 10/10 | 10/10 passing | 0 | 92.83% | 78.57% | 7.9s | $0.0056 | – |
|
|
283
|
+
| `camelcase@8.0.0` index.js | 1/1 | 6/6 | 5/6 | 6/6 passing | 1 | 98.18% | 91.48% | 29.9s | $0.0053 | – |
|
|
284
|
+
| `slugify@1.6.6` slugify.js | 1/1 | 5/5 | 4/5 | 5/5 passing | 2 | 86.95% | 92% | 13.9s | $0.01 | – |
|
|
285
|
+
| `semver@7.6.3` functions/inc.js | 1/1 | 5/6 | 4/5 | 5/5 passing | 1 | 100% | 75% | 10.8s | $0.0037 | – |
|
|
286
|
+
| `dotenv@16.4.7` lib/main.js | 3/7 | 12/25 | 11/12 | 12/12 passing | 1 | 36.56% | 82.35% | 22.2s | $0.02 | fs, os, crypto |
|
|
287
|
+
|
|
288
|
+
| Metric | Result |
|
|
289
|
+
| --- | --- |
|
|
290
|
+
| Test generation (files with valid tests) | 9/9 (100%) |
|
|
291
|
+
| Test generation (cases passing validation) | 73/88 (83%) |
|
|
292
|
+
| Execution (files whose tests ran) | 9/9 (100%) |
|
|
293
|
+
| Execution (tests passing on first run) | 64/73 (87.7%) |
|
|
294
|
+
| Repair (initially failing tests fixed) | 9/9 (100%) |
|
|
295
|
+
| Repair (flagged as source bugs / set aside) | 0 / 0 |
|
|
296
|
+
| Final (tests passing) | 73/73 (100%) |
|
|
297
|
+
| Files with every test passing (none skipped) | 9/9 |
|
|
298
|
+
| Line coverage, before → after (mean) | 0% → 88.6% |
|
|
299
|
+
| Branch coverage after (mean) | 83.3% |
|
|
300
|
+
| Runtime (total / median per file) | 132.6s / 13.9s |
|
|
301
|
+
| Model cost (total / mean per file) | $0.07 / $0.0080 |
|
|
302
|
+
| Tokens (in / out) | 45,040 / 8,485 |
|
|
303
|
+
|
|
304
|
+
How to read it:
|
|
305
|
+
|
|
306
|
+
- "Coverage before" is 0% because published npm packages don't include their tests. The comparison is with having no tests, not with each project's own suite.
|
|
307
|
+
- Model output varies between runs. In six full runs during development, the final pass rate ranged from 82.7% to 100%, and mean line coverage from 88.6% to 95%. Rerun the benchmark rather than relying on one number.
|
|
308
|
+
- Each target is one file, and none is large (the biggest is `dotenv/lib/main.js`, 360 lines). Nine files are enough to find failure patterns, not to estimate a precise success rate.
|
|
309
|
+
|
|
310
|
+
Limitations found by the benchmark:
|
|
311
|
+
|
|
312
|
+
- **Functions that call each other in the same file are hard to isolate.** In this run the model wrote 13 `dotenv` cases that spied on `parse` or `configDotenv` to test their callers. A spy can't intercept a call made inside the same file, so validation rejected all 13. Four of the seven functions ended up without tests, and line coverage for the file was 37%. The previous run, where the model tested those functions directly, reached 84%.
|
|
313
|
+
- **Branch coverage trails line coverage** (83% vs. 89%). The model covers typical paths well but misses rarer branches, such as the unit aliases in `ms`.
|
|
314
|
+
- Cases are sometimes rejected for syntax errors or for `require()` calls in the test body. They are dropped rather than retried, which lowers coverage for that file.
|
|
315
|
+
|
|
316
|
+
## Build and link locally
|
|
317
|
+
|
|
318
|
+
```bash
|
|
319
|
+
npm run build
|
|
320
|
+
npm link
|
|
321
|
+
assertloom --help
|
|
322
|
+
```
|
|
323
|
+
|
|
324
|
+
## Commands
|
|
325
|
+
|
|
326
|
+
| Command | Description |
|
|
327
|
+
| --- | --- |
|
|
328
|
+
| `assertloom analyze <file> [--json]` | Statically analyze functions, parameters, imports, and dependencies |
|
|
329
|
+
| `assertloom generate <file> [--provider] [--model] [--base-url] [--out] [--dry-run] [--no-run] [--no-repair] [--max-attempts] [--report] [--mock] [--keep-real] [--allow-network] [--max-cost] [--force]` | Generate Jest tests for exported functions with dependencies mocked, run and repair them, and keep them if they pass |
|
|
330
|
+
| `assertloom run <testFile> [--source] [--timeout] [--allow-network] [--json]` | Run a test file with Jest and report results and coverage |
|
|
331
|
+
| `assertloom ci [--base] [--head] [--files] [--include] [--exclude] [--max-files] [--max-cost] [--max-tokens] [--max-attempts] [--timeout] [--out] [--fail-on] [--dry-run]` | Generate, run, and repair tests for changed files under one shared budget; write `report.json`, `summary.md`, and test previews (used by the [GitHub Action](docs/github-action.md)) |
|
|
332
|
+
| `assertloom help [command]` | Show help |
|
|
333
|
+
|
|
334
|
+
Supported extensions: `.ts`, `.tsx`, `.mts`, `.cts`, `.js`, `.jsx`, `.mjs`, `.cjs`. Missing files, directories, unsupported extensions, existing `*.test.*`/`*.spec.*` files, and files over 1 MB are rejected with exit code 1.
|
|
335
|
+
|
|
336
|
+
## Development
|
|
337
|
+
|
|
338
|
+
```bash
|
|
339
|
+
npm test # Vitest
|
|
340
|
+
npm run typecheck
|
|
341
|
+
npm run build # outputs to dist/
|
|
342
|
+
npm run bench # live benchmark (needs OPENAI_API_KEY)
|
|
343
|
+
```
|
|
344
|
+
|
|
345
|
+
CI (`.github/workflows/ci.yml`) runs the type check, tests, build, and a CLI smoke test on Linux, Windows, and macOS with Node 22 and 24. It also installs the packed tarball into an empty project and runs it.
|
|
346
|
+
|
|
347
|
+
## Project layout
|
|
348
|
+
|
|
349
|
+
```
|
|
350
|
+
src/
|
|
351
|
+
cli/ Commander entry point, commands, input validation
|
|
352
|
+
analyzer/ Babel AST analysis: parser, functions, dependencies, import usage
|
|
353
|
+
ai/ LLM provider interface, OpenAI provider, prompts, secret redaction
|
|
354
|
+
generator/ Import and mock planning, response schema, case validation, test file writing
|
|
355
|
+
runner/ Jest setup detection, isolated Jest runs, result parsing, coverage
|
|
356
|
+
repair/ Failure classification and the repair loop
|
|
357
|
+
ci/ Changed-file selection, the multi-file run with a shared budget, and reports for `assertloom ci`
|
|
358
|
+
types/ Shared types
|
|
359
|
+
action.yml The GitHub Action
|
|
360
|
+
bench/ Benchmark targets, runner, and published results
|
|
361
|
+
docs/ GitHub Action guide and terminal demo recording
|
|
362
|
+
examples/ Sample source files and sample-project/
|
|
363
|
+
scripts/ Demo recorder
|
|
364
|
+
fixtures/ Inputs for Assertloom's own tests
|
|
365
|
+
tests/ Assertloom's own tests (Vitest)
|
|
366
|
+
```
|
|
367
|
+
|
|
368
|
+
## Security principles
|
|
369
|
+
|
|
370
|
+
Assertloom treats target repositories and generated tests as untrusted:
|
|
371
|
+
|
|
372
|
+
- Analysis is static. Target code is never executed during analysis, including the local modules Assertloom reads to decide what to mock.
|
|
373
|
+
- Generated tests run against mocks of packages, databases, the file system, subprocesses, and `fetch`, so they do not reach real services by default.
|
|
374
|
+
- Generated tests run in a separate process with a hard timeout, without credential-like environment variables, and with network connections blocked unless you pass `--allow-network`.
|
|
375
|
+
- Only the target file is sent to the AI provider. `.env` files and other files are never sent, likely secrets (API keys, tokens, private keys, credentials in URLs, quoted `password`/`secret`/`token` values) are redacted first, and OpenAI requests are made with `store: false`.
|
|
376
|
+
- A diff is shown before any existing test file is overwritten, and files that Assertloom did not generate are never overwritten without `--force`.
|
|
377
|
+
- The repair loop only edits generated tests, never production source code.
|
|
378
|
+
- Retries (at most 3 repair rounds), model spending (`--max-cost`, default $1), and execution time (120 seconds per Jest run, 5 seconds per test) have hard limits.
|
|
379
|
+
|
|
380
|
+
Generated tests run as normal Node code on your machine; the network guard and timeouts are safeguards, not a sandbox. Only run Assertloom against code you trust, or run it inside a container or VM.
|
|
381
|
+
|
|
382
|
+
## Roadmap
|
|
383
|
+
|
|
384
|
+
All eight milestones are done: CLI foundation, AST analyzer, AI test generation, execution engine, self-repair, dependency mocking, evaluation and hardening, and the open-source release. [ROADMAP.md](ROADMAP.md) lists what comes next, starting with the weak spots the benchmark found. Changes are listed in [CHANGELOG.md](CHANGELOG.md).
|
|
385
|
+
|
|
386
|
+
## Contributing
|
|
387
|
+
|
|
388
|
+
Bug reports, benchmark results on other projects, and pull requests are welcome. See [CONTRIBUTING.md](CONTRIBUTING.md) for setup and guidelines, and [SECURITY.md](SECURITY.md) for reporting vulnerabilities privately.
|
|
389
|
+
|
|
390
|
+
Releases are published to npm by the [release workflow](.github/workflows/release.yml) when a `v*` tag is pushed, with [npm provenance](https://docs.npmjs.com/generating-provenance-statements) linking each version to the commit and workflow run that built it. If npm stages a version for review (it does for a package's first release), a maintainer approves it with 2FA in the package's Staged Packages tab on npmjs.com or with `npm stage approve <stage-id>` before it becomes installable.
|
|
391
|
+
|
|
392
|
+
## License
|
|
393
|
+
|
|
394
|
+
[MIT](LICENSE)
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { type LLMProvider, type StructuredRequest, type StructuredResponse } from "./provider.js";
|
|
2
|
+
export declare const DEFAULT_ANTHROPIC_MODEL = "claude-haiku-5-5";
|
|
3
|
+
/** Claude through the Messages API, with JSON output constrained to the request schema. */
|
|
4
|
+
export declare class AnthropicProvider implements LLMProvider {
|
|
5
|
+
readonly model: string;
|
|
6
|
+
private readonly fetchImpl;
|
|
7
|
+
readonly name = "anthropic";
|
|
8
|
+
private readonly apiKey;
|
|
9
|
+
constructor(model?: string, apiKey?: string | undefined, fetchImpl?: typeof fetch);
|
|
10
|
+
generateStructured<T>(request: StructuredRequest<T>): Promise<StructuredResponse<T>>;
|
|
11
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { zodTextFormat } from "openai/helpers/zod";
|
|
2
|
+
import { ProviderError } from "./provider.js";
|
|
3
|
+
export const DEFAULT_ANTHROPIC_MODEL = "claude-haiku-5-5";
|
|
4
|
+
const API_URL = "https://api.anthropic.com/v1/messages";
|
|
5
|
+
const TIMEOUT_MS = 120_000;
|
|
6
|
+
/** Claude through the Messages API, with JSON output constrained to the request schema. */
|
|
7
|
+
export class AnthropicProvider {
|
|
8
|
+
model;
|
|
9
|
+
fetchImpl;
|
|
10
|
+
name = "anthropic";
|
|
11
|
+
apiKey;
|
|
12
|
+
constructor(model = DEFAULT_ANTHROPIC_MODEL, apiKey = process.env.ANTHROPIC_API_KEY, fetchImpl = fetch) {
|
|
13
|
+
this.model = model;
|
|
14
|
+
this.fetchImpl = fetchImpl;
|
|
15
|
+
if (!apiKey) {
|
|
16
|
+
throw new ProviderError("ANTHROPIC_API_KEY is not set. Create a key at https://platform.claude.com/settings/keys and set it as an environment variable.");
|
|
17
|
+
}
|
|
18
|
+
this.apiKey = apiKey;
|
|
19
|
+
}
|
|
20
|
+
async generateStructured(request) {
|
|
21
|
+
// The strict schema OpenAI accepts (every property required, no additional properties) also suits Claude.
|
|
22
|
+
const { schema } = zodTextFormat(request.schema, request.schemaName);
|
|
23
|
+
let response;
|
|
24
|
+
try {
|
|
25
|
+
response = await this.fetchImpl(API_URL, {
|
|
26
|
+
method: "POST",
|
|
27
|
+
headers: { "content-type": "application/json", "x-api-key": this.apiKey, "anthropic-version": "2023-06-01" },
|
|
28
|
+
body: JSON.stringify({
|
|
29
|
+
model: this.model,
|
|
30
|
+
max_tokens: request.maxOutputTokens,
|
|
31
|
+
system: request.system,
|
|
32
|
+
messages: [{ role: "user", content: request.prompt }],
|
|
33
|
+
output_config: { format: { type: "json_schema", schema } },
|
|
34
|
+
}),
|
|
35
|
+
signal: AbortSignal.timeout(TIMEOUT_MS),
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
catch (error) {
|
|
39
|
+
throw new ProviderError(`Anthropic request failed: ${error.message}`);
|
|
40
|
+
}
|
|
41
|
+
const body = (await response.json().catch(() => ({})));
|
|
42
|
+
if (!response.ok) {
|
|
43
|
+
throw new ProviderError(`Anthropic request failed (${response.status}): ${body.error?.message ?? response.statusText}`);
|
|
44
|
+
}
|
|
45
|
+
if (body.stop_reason === "max_tokens")
|
|
46
|
+
throw new ProviderError("Anthropic response was cut off (max_tokens).");
|
|
47
|
+
if (body.stop_reason === "refusal")
|
|
48
|
+
throw new ProviderError("Claude declined the request.");
|
|
49
|
+
const text = body.content?.find((block) => block.type === "text")?.text;
|
|
50
|
+
let json;
|
|
51
|
+
try {
|
|
52
|
+
json = JSON.parse(text ?? "");
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
throw new ProviderError("Anthropic returned no structured output.");
|
|
56
|
+
}
|
|
57
|
+
const parsed = request.schema.safeParse(json);
|
|
58
|
+
if (!parsed.success) {
|
|
59
|
+
throw new ProviderError(`Anthropic returned output that does not match the ${request.schemaName} schema: ${parsed.error.message}`);
|
|
60
|
+
}
|
|
61
|
+
return {
|
|
62
|
+
data: parsed.data,
|
|
63
|
+
model: body.model ?? this.model,
|
|
64
|
+
usage: { inputTokens: body.usage?.input_tokens ?? 0, outputTokens: body.usage?.output_tokens ?? 0 },
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
//# sourceMappingURL=anthropic.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"anthropic.js","sourceRoot":"","sources":["../../src/ai/anthropic.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,aAAa,EAAqE,MAAM,eAAe,CAAC;AAEjH,MAAM,CAAC,MAAM,uBAAuB,GAAG,kBAAkB,CAAC;AAC1D,MAAM,OAAO,GAAG,uCAAuC,CAAC;AACxD,MAAM,UAAU,GAAG,OAAO,CAAC;AAU3B,2FAA2F;AAC3F,MAAM,OAAO,iBAAiB;IAKjB,KAAK;IAEG,SAAS;IANnB,IAAI,GAAG,WAAW,CAAC;IACX,MAAM,CAAS;IAEhC,YACW,KAAK,GAAW,uBAAuB,EAChD,MAAM,GAAuB,OAAO,CAAC,GAAG,CAAC,iBAAiB,EACzC,SAAS,GAAiB,KAAK;qBAFvC,KAAK;yBAEG,SAAS;QAE1B,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,aAAa,CACrB,gIAAgI,CACjI,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAI,OAA6B;QACvD,0GAA0G;QAC1G,MAAM,EAAE,MAAM,EAAE,GAAG,aAAa,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;QAErE,IAAI,QAAkB,CAAC;QACvB,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE;gBACvC,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,WAAW,EAAE,IAAI,CAAC,MAAM,EAAE,mBAAmB,EAAE,YAAY,EAAE;gBAC5G,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,UAAU,EAAE,OAAO,CAAC,eAAe;oBACnC,MAAM,EAAE,OAAO,CAAC,MAAM;oBACtB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;oBACrD,aAAa,EAAE,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE;iBAC3D,CAAC;gBACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,UAAU,CAAC;aACxC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,aAAa,CAAC,6BAA8B,KAAe,CAAC,OAAO,EAAE,CAAC,CAAC;QACnF,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAqB,CAAC;QAC3E,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,aAAa,CAAC,6BAA6B,QAAQ,CAAC,MAAM,MAAM,IAAI,CAAC,KAAK,EAAE,OAAO,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QAC1H,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,KAAK,YAAY;YAAE,MAAM,IAAI,aAAa,CAAC,8CAA8C,CAAC,CAAC;QAC/G,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS;YAAE,MAAM,IAAI,aAAa,CAAC,8BAA8B,CAAC,CAAC;QAE5F,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC,EAAE,IAAI,CAAC;QACxE,IAAI,IAAa,CAAC;QAClB,IAAI,CAAC;YACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QAChC,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,aAAa,CAAC,0CAA0C,CAAC,CAAC;QACtE,CAAC;QACD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,aAAa,CAAC,qDAAqD,OAAO,CAAC,UAAU,YAAY,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACrI,CAAC;QAED,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK;YAC/B,KAAK,EAAE,EAAE,WAAW,EAAE,IAAI,CAAC,KAAK,EAAE,YAAY,IAAI,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,KAAK,EAAE,aAAa,IAAI,CAAC,EAAE;SACpG,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { type LLMProvider } from "./provider.js";
|
|
2
|
+
export * from "./provider.js";
|
|
3
|
+
export { AnthropicProvider, DEFAULT_ANTHROPIC_MODEL } from "./anthropic.js";
|
|
4
|
+
export { DEFAULT_OPENAI_MODEL, OpenAIProvider } from "./openai.js";
|
|
5
|
+
export { OpenAICompatibleProvider } from "./openaiCompatible.js";
|
|
6
|
+
export declare const PROVIDERS: readonly ["openai", "anthropic", "openai-compatible"];
|
|
7
|
+
export interface ProviderOptions {
|
|
8
|
+
model?: string;
|
|
9
|
+
/** Server URL for `openai-compatible`. */
|
|
10
|
+
baseUrl?: string;
|
|
11
|
+
}
|
|
12
|
+
export declare function createProvider(name: string, options?: ProviderOptions): LLMProvider;
|
package/dist/ai/index.js
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { AnthropicProvider } from "./anthropic.js";
|
|
2
|
+
import { OpenAIProvider } from "./openai.js";
|
|
3
|
+
import { OpenAICompatibleProvider } from "./openaiCompatible.js";
|
|
4
|
+
import { ProviderError } from "./provider.js";
|
|
5
|
+
export * from "./provider.js";
|
|
6
|
+
export { AnthropicProvider, DEFAULT_ANTHROPIC_MODEL } from "./anthropic.js";
|
|
7
|
+
export { DEFAULT_OPENAI_MODEL, OpenAIProvider } from "./openai.js";
|
|
8
|
+
export { OpenAICompatibleProvider } from "./openaiCompatible.js";
|
|
9
|
+
export const PROVIDERS = ["openai", "anthropic", "openai-compatible"];
|
|
10
|
+
export function createProvider(name, options = {}) {
|
|
11
|
+
const model = options.model ?? process.env.ASSERTLOOM_MODEL;
|
|
12
|
+
switch (name) {
|
|
13
|
+
case "openai":
|
|
14
|
+
return new OpenAIProvider(model);
|
|
15
|
+
case "anthropic":
|
|
16
|
+
return new AnthropicProvider(model);
|
|
17
|
+
case "openai-compatible":
|
|
18
|
+
return new OpenAICompatibleProvider(model ?? "", options.baseUrl ?? process.env.ASSERTLOOM_BASE_URL ?? "");
|
|
19
|
+
default:
|
|
20
|
+
throw new ProviderError(`Unknown provider "${name}". Available: ${PROVIDERS.join(", ")}`);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/ai/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,EAAE,aAAa,EAAoB,MAAM,eAAe,CAAC;AAEhE,cAAc,eAAe,CAAC;AAC9B,OAAO,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,gBAAgB,CAAC;AAC5E,OAAO,EAAE,oBAAoB,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AACnE,OAAO,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AAEjE,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,QAAQ,EAAE,WAAW,EAAE,mBAAmB,CAAU,CAAC;AAQ/E,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,OAAO,GAAoB,EAAE;IACxE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;IAC5D,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,QAAQ;YACX,OAAO,IAAI,cAAc,CAAC,KAAK,CAAC,CAAC;QACnC,KAAK,WAAW;YACd,OAAO,IAAI,iBAAiB,CAAC,KAAK,CAAC,CAAC;QACtC,KAAK,mBAAmB;YACtB,OAAO,IAAI,wBAAwB,CAAC,KAAK,IAAI,EAAE,EAAE,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,EAAE,CAAC,CAAC;QAC7G;YACE,MAAM,IAAI,aAAa,CAAC,qBAAqB,IAAI,iBAAiB,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC9F,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { type LLMProvider, type StructuredRequest, type StructuredResponse } from "./provider.js";
|
|
2
|
+
export declare const DEFAULT_OPENAI_MODEL = "gpt-5.4-mini";
|
|
3
|
+
export declare class OpenAIProvider implements LLMProvider {
|
|
4
|
+
readonly model: string;
|
|
5
|
+
readonly name = "openai";
|
|
6
|
+
private readonly client;
|
|
7
|
+
constructor(model?: string, apiKey?: string | undefined);
|
|
8
|
+
generateStructured<T>(request: StructuredRequest<T>): Promise<StructuredResponse<T>>;
|
|
9
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import OpenAI from "openai";
|
|
2
|
+
import { zodTextFormat } from "openai/helpers/zod";
|
|
3
|
+
import { ProviderError } from "./provider.js";
|
|
4
|
+
export const DEFAULT_OPENAI_MODEL = "gpt-5.4-mini";
|
|
5
|
+
export class OpenAIProvider {
|
|
6
|
+
model;
|
|
7
|
+
name = "openai";
|
|
8
|
+
client;
|
|
9
|
+
constructor(model = DEFAULT_OPENAI_MODEL, apiKey = process.env.OPENAI_API_KEY) {
|
|
10
|
+
this.model = model;
|
|
11
|
+
if (!apiKey) {
|
|
12
|
+
throw new ProviderError("OPENAI_API_KEY is not set. Create a key at https://platform.openai.com/api-keys and set it as an environment variable.");
|
|
13
|
+
}
|
|
14
|
+
// Retries would bill a request the spending limit only approved once.
|
|
15
|
+
this.client = new OpenAI({ apiKey, timeout: 120_000, maxRetries: 0 });
|
|
16
|
+
}
|
|
17
|
+
async generateStructured(request) {
|
|
18
|
+
let response;
|
|
19
|
+
try {
|
|
20
|
+
response = await this.client.responses.parse({
|
|
21
|
+
model: this.model,
|
|
22
|
+
input: [
|
|
23
|
+
{ role: "system", content: request.system },
|
|
24
|
+
{ role: "user", content: request.prompt },
|
|
25
|
+
],
|
|
26
|
+
text: { format: zodTextFormat(request.schema, request.schemaName) },
|
|
27
|
+
max_output_tokens: request.maxOutputTokens,
|
|
28
|
+
store: false,
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
catch (error) {
|
|
32
|
+
if (error instanceof OpenAI.APIError) {
|
|
33
|
+
throw new ProviderError(`OpenAI request failed (${error.status ?? "no status"}): ${error.message}`);
|
|
34
|
+
}
|
|
35
|
+
throw new ProviderError(`OpenAI request failed: ${error.message}`);
|
|
36
|
+
}
|
|
37
|
+
if (response.status === "incomplete") {
|
|
38
|
+
throw new ProviderError(`OpenAI response was cut off (${response.incomplete_details?.reason ?? "unknown reason"}).`);
|
|
39
|
+
}
|
|
40
|
+
if (response.output_parsed == null) {
|
|
41
|
+
throw new ProviderError("OpenAI returned no structured output (the model may have refused the request).");
|
|
42
|
+
}
|
|
43
|
+
const parsed = request.schema.safeParse(response.output_parsed);
|
|
44
|
+
if (!parsed.success) {
|
|
45
|
+
throw new ProviderError(`OpenAI returned output that does not match the ${request.schemaName} schema: ${parsed.error.message}`);
|
|
46
|
+
}
|
|
47
|
+
return {
|
|
48
|
+
data: parsed.data,
|
|
49
|
+
model: response.model,
|
|
50
|
+
usage: {
|
|
51
|
+
inputTokens: response.usage?.input_tokens ?? 0,
|
|
52
|
+
outputTokens: response.usage?.output_tokens ?? 0,
|
|
53
|
+
},
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
//# sourceMappingURL=openai.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"openai.js","sourceRoot":"","sources":["../../src/ai/openai.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,aAAa,EAAqE,MAAM,eAAe,CAAC;AAEjH,MAAM,CAAC,MAAM,oBAAoB,GAAG,cAAc,CAAC;AAEnD,MAAM,OAAO,cAAc;IAKd,KAAK;IAJP,IAAI,GAAG,QAAQ,CAAC;IACR,MAAM,CAAS;IAEhC,YACW,KAAK,GAAW,oBAAoB,EAC7C,MAAM,GAAuB,OAAO,CAAC,GAAG,CAAC,cAAc;qBAD9C,KAAK;QAGd,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,aAAa,CACrB,wHAAwH,CACzH,CAAC;QACJ,CAAC;QACD,sEAAsE;QACtE,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAI,OAA6B;QACvD,IAAI,QAAQ,CAAC;QACb,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC;gBAC3C,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,KAAK,EAAE;oBACL,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE;oBAC3C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE;iBAC1C;gBACD,IAAI,EAAE,EAAE,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,UAAU,CAAC,EAAE;gBACnE,iBAAiB,EAAE,OAAO,CAAC,eAAe;gBAC1C,KAAK,EAAE,KAAK;aACb,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,MAAM,CAAC,QAAQ,EAAE,CAAC;gBACrC,MAAM,IAAI,aAAa,CAAC,0BAA0B,KAAK,CAAC,MAAM,IAAI,WAAW,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YACtG,CAAC;YACD,MAAM,IAAI,aAAa,CAAC,0BAA2B,KAAe,CAAC,OAAO,EAAE,CAAC,CAAC;QAChF,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;YACrC,MAAM,IAAI,aAAa,CAAC,gCAAgC,QAAQ,CAAC,kBAAkB,EAAE,MAAM,IAAI,gBAAgB,IAAI,CAAC,CAAC;QACvH,CAAC;QACD,IAAI,QAAQ,CAAC,aAAa,IAAI,IAAI,EAAE,CAAC;YACnC,MAAM,IAAI,aAAa,CAAC,gFAAgF,CAAC,CAAC;QAC5G,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;QAChE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,aAAa,CAAC,kDAAkD,OAAO,CAAC,UAAU,YAAY,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAClI,CAAC;QAED,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,KAAK,EAAE;gBACL,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY,IAAI,CAAC;gBAC9C,YAAY,EAAE,QAAQ,CAAC,KAAK,EAAE,aAAa,IAAI,CAAC;aACjD;SACF,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { type LLMProvider, type StructuredRequest, type StructuredResponse } from "./provider.js";
|
|
2
|
+
/**
|
|
3
|
+
* Any server that speaks the OpenAI Chat Completions API with JSON-schema response formats:
|
|
4
|
+
* Ollama, LM Studio, vLLM, llama.cpp, OpenRouter, and others.
|
|
5
|
+
*/
|
|
6
|
+
export declare class OpenAICompatibleProvider implements LLMProvider {
|
|
7
|
+
readonly model: string;
|
|
8
|
+
readonly baseURL: string;
|
|
9
|
+
readonly name = "openai-compatible";
|
|
10
|
+
private readonly client;
|
|
11
|
+
constructor(model: string, baseURL: string, apiKey?: string | undefined);
|
|
12
|
+
generateStructured<T>(request: StructuredRequest<T>): Promise<StructuredResponse<T>>;
|
|
13
|
+
}
|