@lumozai/skim 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +112 -2
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +101 -0
- package/dist/cli.js.map +1 -0
- package/dist/core/registry.d.ts +8 -0
- package/dist/core/registry.js +41 -0
- package/dist/core/registry.js.map +1 -0
- package/dist/core/runner.d.ts +14 -0
- package/dist/core/runner.js +83 -0
- package/dist/core/runner.js.map +1 -0
- package/dist/core/store.d.ts +7 -0
- package/dist/core/store.js +158 -0
- package/dist/core/store.js.map +1 -0
- package/dist/core/tokens.d.ts +5 -0
- package/dist/core/tokens.js +41 -0
- package/dist/core/tokens.js.map +1 -0
- package/dist/core/types.d.ts +62 -0
- package/dist/core/types.js +2 -0
- package/dist/core/types.js.map +1 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.js +6 -0
- package/dist/index.js.map +1 -0
- package/dist/mcp.d.ts +1 -0
- package/dist/mcp.js +64 -0
- package/dist/mcp.js.map +1 -0
- package/dist/parsers/eslint.d.ts +2 -0
- package/dist/parsers/eslint.js +69 -0
- package/dist/parsers/eslint.js.map +1 -0
- package/dist/parsers/generic.d.ts +2 -0
- package/dist/parsers/generic.js +65 -0
- package/dist/parsers/generic.js.map +1 -0
- package/dist/parsers/git.d.ts +2 -0
- package/dist/parsers/git.js +157 -0
- package/dist/parsers/git.js.map +1 -0
- package/dist/parsers/index.d.ts +10 -0
- package/dist/parsers/index.js +18 -0
- package/dist/parsers/index.js.map +1 -0
- package/dist/parsers/pkg-install.d.ts +2 -0
- package/dist/parsers/pkg-install.js +37 -0
- package/dist/parsers/pkg-install.js.map +1 -0
- package/dist/parsers/pytest.d.ts +2 -0
- package/dist/parsers/pytest.js +92 -0
- package/dist/parsers/pytest.js.map +1 -0
- package/dist/parsers/tsc.d.ts +2 -0
- package/dist/parsers/tsc.js +57 -0
- package/dist/parsers/tsc.js.map +1 -0
- package/dist/parsers/vitest-jest.d.ts +2 -0
- package/dist/parsers/vitest-jest.js +106 -0
- package/dist/parsers/vitest-jest.js.map +1 -0
- package/package.json +54 -4
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 LumozAI
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,113 @@
|
|
|
1
|
-
#
|
|
1
|
+
# skim
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
> **Skim first, zoom in only when needed.**
|
|
4
|
+
> Token-efficient CLI & MCP output compressor with lossless-on-demand drill-down for AI coding agents.
|
|
5
|
+
> *An open-source project by LumozAI.*
|
|
6
|
+
|
|
7
|
+
Coding agents burn thousands of tokens reading passing tests, `node_modules` stack frames, `npm install` warnings, and lockfile diffs. **`skim`** sits between your agent and your shell — compressing noisy tool output down to the few lines that matter, while saving the raw log locally under a `run_id` so the agent can drill in *only* when it actually needs more detail.
|
|
8
|
+
|
|
9
|
+
- **100% Local & Private:** Zero telemetry, zero accounts, zero cloud calls. Your code never leaves your machine.
|
|
10
|
+
- **Lossless-on-Demand:** Never forces an agent to re-run a slow test suite just because a line was trimmed. Every run returns a `run_id` for instant `inspect_log` drill-down.
|
|
11
|
+
- **Drop-in MCP + CLI + SDK:** Works with Claude Code, Cursor, Codex, Windsurf, Antigravity, or directly inside your own agent harness.
|
|
12
|
+
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
## Benchmark Results
|
|
16
|
+
|
|
17
|
+
Token counts use the real `o200k_base` tokenizer (GPT-4o family; a close proxy for Claude), and include skim's footer.
|
|
18
|
+
|
|
19
|
+
**Real command output** (captured from actual runs, not fixtures):
|
|
20
|
+
|
|
21
|
+
| Command | Raw Tokens | Skim Tokens | Saved |
|
|
22
|
+
| :--- | ---: | ---: | ---: |
|
|
23
|
+
| `npm install --loglevel verbose` (4 deps) | 9,339 | 36 | **-99.6%** |
|
|
24
|
+
| `pytest -v` (203 tests, 3 failing) | 3,889 | 168 | **-95.7%** |
|
|
25
|
+
| `pytest` default (203 tests, 3 failing) | 441 | 168 | **-61.9%** |
|
|
26
|
+
| `tsc --pretty` (4 errors) | 510 | 132 | **-74.1%** |
|
|
27
|
+
| `tsc` plain (4 errors) | 130 | 135 | passthrough (+6-token exit marker) |
|
|
28
|
+
|
|
29
|
+
Savings depend on how noisy the command is: verbose logs and big test suites compress the most, while output that is already short is passed through unchanged rather than rewritten.
|
|
30
|
+
|
|
31
|
+
**Synthetic fixtures** (`npm run bench`):
|
|
32
|
+
|
|
33
|
+
| Scenario | Parser | Raw Tokens | Skim Tokens | Saved |
|
|
34
|
+
| :--- | :--- | ---: | ---: | ---: |
|
|
35
|
+
| Vitest (2 fails / 48 pass) | `vitest-jest` | 1,569 | 230 | **-85.3%** |
|
|
36
|
+
| Pytest (1 fail / 60 pass) | `pytest` | 1,288 | 97 | **-92.5%** |
|
|
37
|
+
| TypeScript (`tsc --pretty`) | `tsc` | 314 | 126 | **-59.9%** |
|
|
38
|
+
| ESLint (3 errors, 18 warnings) | `eslint` | 502 | 117 | **-76.7%** |
|
|
39
|
+
| npm install (noisy) | `pkg-install` | 532 | 37 | **-93%** |
|
|
40
|
+
| git diff (code + lockfile) | `git` | 5,915 | 85 | **-98.6%** |
|
|
41
|
+
|
|
42
|
+
### Passthrough rule
|
|
43
|
+
|
|
44
|
+
skim only uses its compressed summary when it is at least 10% smaller than the raw output, footer included (`MIN_SAVING_RATIO`). Otherwise the agent gets the raw output (ANSI-stripped) plus `[exit N]` when the command failed. So skim never meaningfully increases token use.
|
|
45
|
+
|
|
46
|
+
> **New to `skim`?** Read the step-by-step **[Simple User Manual](./USER_MANUAL.md)** for a plain-English setup and usage guide.
|
|
47
|
+
|
|
48
|
+
---
|
|
49
|
+
|
|
50
|
+
## Quick Start (MCP Server)
|
|
51
|
+
|
|
52
|
+
Add `skim` to your MCP client config (Claude Code, Cursor, Codex, etc.):
|
|
53
|
+
|
|
54
|
+
```json
|
|
55
|
+
{
|
|
56
|
+
"mcpServers": {
|
|
57
|
+
"skim": {
|
|
58
|
+
"command": "npx",
|
|
59
|
+
"args": ["-y", "@lumozai/skim", "mcp"]
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
### Exposed MCP Tools
|
|
66
|
+
1. **`run_compressed`** — Executes a shell command, auto-detects the output format (`vitest`/`jest`, `pytest`, `tsc`, `eslint`, `npm`/`pnpm`/`yarn`/`pip`, `git`, or smart generic fallback), returns a compact summary + `run_id`.
|
|
67
|
+
2. **`inspect_log`** — Searches or slices the saved raw output for a `run_id` by `--query` or line range (`startLine`/`endLine`).
|
|
68
|
+
|
|
69
|
+
---
|
|
70
|
+
|
|
71
|
+
## CLI Usage
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
# Run any command with compressed output
|
|
75
|
+
skim run npx vitest run
|
|
76
|
+
skim run pytest -v
|
|
77
|
+
skim run npm install
|
|
78
|
+
|
|
79
|
+
# Drill into a saved raw log only if needed
|
|
80
|
+
skim inspect run_a1b2c3 --query "AssertionError" --context 3
|
|
81
|
+
skim inspect run_a1b2c3 --start 1 --end 50
|
|
82
|
+
|
|
83
|
+
# View cumulative local token savings
|
|
84
|
+
skim stats
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
---
|
|
88
|
+
|
|
89
|
+
## Programmatic SDK Usage
|
|
90
|
+
|
|
91
|
+
Use `skim` directly inside your own coding agent harness:
|
|
92
|
+
|
|
93
|
+
```ts
|
|
94
|
+
import { runCompressed, inspectRun, compressOutput } from "@lumozai/skim";
|
|
95
|
+
|
|
96
|
+
const result = await runCompressed("npx vitest run", { cwd: "/path/to/repo" });
|
|
97
|
+
console.log(result.summary);
|
|
98
|
+
console.log(`Saved ${result.savedPercent}% (${result.rawTokens} -> ${result.compressedTokens} tokens)`);
|
|
99
|
+
|
|
100
|
+
// Drill down if the agent requests more detail
|
|
101
|
+
const detail = inspectRun(result.runId!, { query: "TypeError", contextLines: 3 });
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
---
|
|
105
|
+
|
|
106
|
+
## Roadmap
|
|
107
|
+
- [x] **v0.1:** Output compressor (`vitest`/`jest`, `pytest`, `tsc`, `eslint`, `pkg-install`, `git`, `generic`) + lossless-on-demand `inspect_log` + MCP server + CLI + benchmarks
|
|
108
|
+
- [ ] **v0.2:** Code zoom layer (`repo_map`, `read_skeleton`, `read_symbol`) via tree-sitter
|
|
109
|
+
- [ ] **v0.3:** Session read cache (avoid re-sending unchanged files in long agent sessions)
|
|
110
|
+
- [ ] **v0.4:** Additional parsers (`cargo test`, `go test`, `phpunit`, `next build`, `docker build`)
|
|
111
|
+
|
|
112
|
+
## License
|
|
113
|
+
MIT © LumozAI
|
package/dist/cli.d.ts
ADDED
package/dist/cli.js
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { formatSkimOutput, runCompressed } from "./core/runner.js";
|
|
3
|
+
import { inspectRun, loadStats } from "./core/store.js";
|
|
4
|
+
import { startMcpServer } from "./mcp.js";
|
|
5
|
+
function printHelp() {
|
|
6
|
+
console.log(`skim - Token-efficient CLI & MCP output compressor for AI coding agents
|
|
7
|
+
|
|
8
|
+
Usage:
|
|
9
|
+
skim run <command...> Run a command and print compressed output
|
|
10
|
+
skim inspect <run_id> [options] Inspect saved raw output from a run
|
|
11
|
+
--query <text> Search matching lines (case-insensitive)
|
|
12
|
+
--start <line> 1-based start line
|
|
13
|
+
--end <line> 1-based end line
|
|
14
|
+
--context <n> Context lines around query matches (default: 2)
|
|
15
|
+
skim mcp Start the MCP stdio server
|
|
16
|
+
skim stats Show cumulative local token savings
|
|
17
|
+
skim --help Show this help message`);
|
|
18
|
+
}
|
|
19
|
+
async function main() {
|
|
20
|
+
const args = process.argv.slice(2);
|
|
21
|
+
const sub = args[0];
|
|
22
|
+
if (!sub || sub === "-h" || sub === "--help") {
|
|
23
|
+
printHelp();
|
|
24
|
+
process.exit(0);
|
|
25
|
+
}
|
|
26
|
+
if (sub === "mcp") {
|
|
27
|
+
await startMcpServer();
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
if (sub === "run") {
|
|
31
|
+
const cmd = args.slice(1).join(" ").trim();
|
|
32
|
+
if (!cmd) {
|
|
33
|
+
console.error("Error: missing command. Example: skim run npm test");
|
|
34
|
+
process.exit(1);
|
|
35
|
+
}
|
|
36
|
+
const result = await runCompressed(cmd);
|
|
37
|
+
console.log(formatSkimOutput(result));
|
|
38
|
+
process.exit(result.exitCode);
|
|
39
|
+
}
|
|
40
|
+
if (sub === "inspect") {
|
|
41
|
+
const runId = args[1];
|
|
42
|
+
if (!runId) {
|
|
43
|
+
console.error("Error: missing run_id. Example: skim inspect run_a1b2c3 --query Error");
|
|
44
|
+
process.exit(1);
|
|
45
|
+
}
|
|
46
|
+
let query;
|
|
47
|
+
let startLine;
|
|
48
|
+
let endLine;
|
|
49
|
+
let contextLines;
|
|
50
|
+
for (let i = 2; i < args.length; i++) {
|
|
51
|
+
if (args[i] === "--query" && args[i + 1]) {
|
|
52
|
+
query = args[++i];
|
|
53
|
+
}
|
|
54
|
+
else if (args[i] === "--start" && args[i + 1]) {
|
|
55
|
+
startLine = Number(args[++i]);
|
|
56
|
+
}
|
|
57
|
+
else if (args[i] === "--end" && args[i + 1]) {
|
|
58
|
+
endLine = Number(args[++i]);
|
|
59
|
+
}
|
|
60
|
+
else if (args[i] === "--context" && args[i + 1]) {
|
|
61
|
+
contextLines = Number(args[++i]);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
const res = inspectRun(runId, { query, startLine, endLine, contextLines });
|
|
65
|
+
if (!res) {
|
|
66
|
+
console.error(`Run "${runId}" not found.`);
|
|
67
|
+
process.exit(1);
|
|
68
|
+
}
|
|
69
|
+
console.log(`[${res.runId} | ${res.command} | ${res.returnedLines}/${res.totalLines} lines]`);
|
|
70
|
+
console.log(res.content);
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
if (sub === "stats") {
|
|
74
|
+
const stats = loadStats();
|
|
75
|
+
const pct = stats.totalRawTokens > 0
|
|
76
|
+
? ((stats.totalSavedTokens / stats.totalRawTokens) * 100).toFixed(1)
|
|
77
|
+
: "0.0";
|
|
78
|
+
console.log(`skim local stats:`);
|
|
79
|
+
console.log(` Runs: ${stats.totalRuns}`);
|
|
80
|
+
console.log(` Raw tokens: ${stats.totalRawTokens.toLocaleString()}`);
|
|
81
|
+
console.log(` Compressed: ${stats.totalCompressedTokens.toLocaleString()}`);
|
|
82
|
+
console.log(` Saved: ${stats.totalSavedTokens.toLocaleString()} (${pct}%)`);
|
|
83
|
+
const entries = Object.entries(stats.byParser);
|
|
84
|
+
if (entries.length > 0) {
|
|
85
|
+
console.log(`\nBy parser:`);
|
|
86
|
+
for (const [name, p] of entries) {
|
|
87
|
+
const saved = Math.max(0, p.rawTokens - p.compressedTokens);
|
|
88
|
+
const pPct = p.rawTokens > 0 ? ((saved / p.rawTokens) * 100).toFixed(1) : "0.0";
|
|
89
|
+
console.log(` ${name.padEnd(14)} ${p.runs} runs | -${saved.toLocaleString()} tok (-${pPct}%)`);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
console.error(`Unknown subcommand "${sub}". Run "skim --help" for usage.`);
|
|
95
|
+
process.exit(1);
|
|
96
|
+
}
|
|
97
|
+
main().catch((err) => {
|
|
98
|
+
console.error(err);
|
|
99
|
+
process.exit(1);
|
|
100
|
+
});
|
|
101
|
+
//# sourceMappingURL=cli.js.map
|
package/dist/cli.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACnE,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAE1C,SAAS,SAAS;IAChB,OAAO,CAAC,GAAG,CAAC;;;;;;;;;;;4DAW8C,CAAC,CAAC;AAC9D,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IAEpB,IAAI,CAAC,GAAG,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC7C,SAAS,EAAE,CAAC;QACZ,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;QAClB,MAAM,cAAc,EAAE,CAAC;QACvB,OAAO;IACT,CAAC;IAED,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAC3C,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,OAAO,CAAC,KAAK,CAAC,oDAAoD,CAAC,CAAC;YACpE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,GAAG,CAAC,CAAC;QACxC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC;QACtC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IAED,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACtB,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,uEAAuE,CAAC,CAAC;YACvF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,IAAI,KAAyB,CAAC;QAC9B,IAAI,SAA6B,CAAC;QAClC,IAAI,OAA2B,CAAC;QAChC,IAAI,YAAgC,CAAC;QAErC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBACzC,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;YACpB,CAAC;iBAAM,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBAChD,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAChC,CAAC;iBAAM,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,OAAO,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBAC9C,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAC9B,CAAC;iBAAM,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,WAAW,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBAClD,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;QAED,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;QAC3E,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,OAAO,CAAC,KAAK,CAAC,QAAQ,KAAK,cAAc,CAAC,CAAC;YAC3C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,KAAK,MAAM,GAAG,CAAC,OAAO,MAAM,GAAG,CAAC,aAAa,IAAI,GAAG,CAAC,UAAU,SAAS,CAAC,CAAC;QAC9F,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACzB,OAAO;IACT,CAAC;IAED,IAAI,GAAG,KAAK,OAAO,EAAE,CAAC;QACpB,MAAM,KAAK,GAAG,SAAS,EAAE,CAAC;QAC1B,MAAM,GAAG,GACP,KAAK,CAAC,cAAc,GAAG,CAAC;YACtB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,gBAAgB,GAAG,KAAK,CAAC,cAAc,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YACpE,CAAC,CAAC,KAAK,CAAC;QACZ,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACjC,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;QACjD,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,CAAC,cAAc,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QACvE,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,CAAC,qBAAqB,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QAC9E,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,CAAC,gBAAgB,CAAC,cAAc,EAAE,KAAK,GAAG,IAAI,CAAC,CAAC;QACnF,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC/C,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;YAC5B,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,OAAO,EAAE,CAAC;gBAChC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,gBAAgB,CAAC,CAAC;gBAC5D,MAAM,IAAI,GAAG,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;gBAChF,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,YAAY,KAAK,CAAC,cAAc,EAAE,UAAU,IAAI,IAAI,CAAC,CAAC;YAClG,CAAC;QACH,CAAC;QACD,OAAO;IACT,CAAC;IAED,OAAO,CAAC,KAAK,CAAC,uBAAuB,GAAG,iCAAiC,CAAC,CAAC;IAC3E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { CompressInput, CompressResult, OutputParser } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Compression is only used when it beats the raw output by at least this
|
|
4
|
+
* fraction, including the footer. Below that, the agent gets the (ANSI-stripped)
|
|
5
|
+
* raw output: it's cheaper or about the same, and nothing is lost.
|
|
6
|
+
*/
|
|
7
|
+
export declare const MIN_SAVING_RATIO = 0.1;
|
|
8
|
+
export declare function compressOutput(input: CompressInput, parsers?: OutputParser[]): CompressResult;
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { builtinParsers } from "../parsers/index.js";
|
|
2
|
+
import { countLines, countTokens, stripAnsi } from "./tokens.js";
|
|
3
|
+
/**
|
|
4
|
+
* Compression is only used when it beats the raw output by at least this
|
|
5
|
+
* fraction, including the footer. Below that, the agent gets the (ANSI-stripped)
|
|
6
|
+
* raw output: it's cheaper or about the same, and nothing is lost.
|
|
7
|
+
*/
|
|
8
|
+
export const MIN_SAVING_RATIO = 0.1;
|
|
9
|
+
/** Representative footer, used to charge the footer's cost before a run id exists. */
|
|
10
|
+
const FOOTER_TEMPLATE = "[skim run_000000 · 9999 lines · exit 1]";
|
|
11
|
+
export function compressOutput(input, parsers = builtinParsers) {
|
|
12
|
+
const combinedRaw = [input.stdout, input.stderr].filter(Boolean).join("\n");
|
|
13
|
+
const cleanOutput = stripAnsi(combinedRaw);
|
|
14
|
+
const parser = parsers.find((p) => p.match(input.command, cleanOutput)) ??
|
|
15
|
+
parsers[parsers.length - 1];
|
|
16
|
+
const parsed = parser.parse(input, cleanOutput).trim();
|
|
17
|
+
const rawTokens = countTokens(combinedRaw);
|
|
18
|
+
const footerTokens = countTokens(FOOTER_TEMPLATE);
|
|
19
|
+
const parsedTotal = countTokens(parsed) + footerTokens;
|
|
20
|
+
// Pass through when compressing wouldn't really save anything.
|
|
21
|
+
const passthrough = parsedTotal > rawTokens * (1 - MIN_SAVING_RATIO);
|
|
22
|
+
const summary = passthrough ? cleanOutput.trim() : parsed;
|
|
23
|
+
const compressedTokens = passthrough
|
|
24
|
+
? countTokens(summary) + (input.exitCode !== 0 ? countTokens("[exit 1]") : 0)
|
|
25
|
+
: parsedTotal;
|
|
26
|
+
const savedTokens = Math.max(0, rawTokens - compressedTokens);
|
|
27
|
+
const savedPercent = rawTokens > 0 ? Math.round((savedTokens / rawTokens) * 1000) / 10 : 0;
|
|
28
|
+
return {
|
|
29
|
+
parser: parser.name,
|
|
30
|
+
summary,
|
|
31
|
+
passthrough,
|
|
32
|
+
rawTokens,
|
|
33
|
+
compressedTokens,
|
|
34
|
+
savedTokens,
|
|
35
|
+
savedPercent,
|
|
36
|
+
rawLines: countLines(cleanOutput),
|
|
37
|
+
compressedLines: countLines(summary),
|
|
38
|
+
exitCode: input.exitCode,
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
//# sourceMappingURL=registry.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"registry.js","sourceRoot":"","sources":["../../src/core/registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAGjE;;;;GAIG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAEpC,sFAAsF;AACtF,MAAM,eAAe,GAAG,yCAAyC,CAAC;AAElE,MAAM,UAAU,cAAc,CAC5B,KAAoB,EACpB,UAA0B,cAAc;IAExC,MAAM,WAAW,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5E,MAAM,WAAW,GAAG,SAAS,CAAC,WAAW,CAAC,CAAC;IAE3C,MAAM,MAAM,GACV,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QACxD,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAE9B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC,IAAI,EAAE,CAAC;IACvD,MAAM,SAAS,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;IAC3C,MAAM,YAAY,GAAG,WAAW,CAAC,eAAe,CAAC,CAAC;IAClD,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,GAAG,YAAY,CAAC;IAEvD,+DAA+D;IAC/D,MAAM,WAAW,GAAG,WAAW,GAAG,SAAS,GAAG,CAAC,CAAC,GAAG,gBAAgB,CAAC,CAAC;IACrE,MAAM,OAAO,GAAG,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IAC1D,MAAM,gBAAgB,GAAG,WAAW;QAClC,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7E,CAAC,CAAC,WAAW,CAAC;IAEhB,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,gBAAgB,CAAC,CAAC;IAC9D,MAAM,YAAY,GAChB,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAExE,OAAO;QACL,MAAM,EAAE,MAAM,CAAC,IAAI;QACnB,OAAO;QACP,WAAW;QACX,SAAS;QACT,gBAAgB;QAChB,WAAW;QACX,YAAY;QACZ,QAAQ,EAAE,UAAU,CAAC,WAAW,CAAC;QACjC,eAAe,EAAE,UAAU,CAAC,OAAO,CAAC;QACpC,QAAQ,EAAE,KAAK,CAAC,QAAQ;KACzB,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { CompressResult } from "./types.js";
|
|
2
|
+
export interface RunCommandOptions {
|
|
3
|
+
cwd?: string;
|
|
4
|
+
timeoutMs?: number;
|
|
5
|
+
env?: NodeJS.ProcessEnv;
|
|
6
|
+
saveRaw?: boolean;
|
|
7
|
+
}
|
|
8
|
+
export declare function runCompressed(command: string, options?: RunCommandOptions): Promise<CompressResult>;
|
|
9
|
+
/**
|
|
10
|
+
* Formats a result for the agent. The footer is kept short because it is paid
|
|
11
|
+
* on every call: run id (for `skim inspect`), raw size, and exit code.
|
|
12
|
+
* Token stats are left out of the footer; they are in `skim stats`.
|
|
13
|
+
*/
|
|
14
|
+
export declare function formatSkimOutput(result: CompressResult): string;
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { compressOutput } from "./registry.js";
|
|
3
|
+
import { generateRunId, saveRun } from "./store.js";
|
|
4
|
+
import { stripAnsi } from "./tokens.js";
|
|
5
|
+
export function runCompressed(command, options = {}) {
|
|
6
|
+
const cwd = options.cwd || process.cwd();
|
|
7
|
+
const timeoutMs = options.timeoutMs ?? 120_000;
|
|
8
|
+
const saveRaw = options.saveRaw ?? true;
|
|
9
|
+
return new Promise((resolve) => {
|
|
10
|
+
const child = spawn(command, {
|
|
11
|
+
cwd,
|
|
12
|
+
shell: true,
|
|
13
|
+
env: { ...process.env, ...options.env },
|
|
14
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
15
|
+
});
|
|
16
|
+
let stdout = "";
|
|
17
|
+
let stderr = "";
|
|
18
|
+
let timedOut = false;
|
|
19
|
+
const timer = setTimeout(() => {
|
|
20
|
+
timedOut = true;
|
|
21
|
+
child.kill("SIGTERM");
|
|
22
|
+
}, timeoutMs);
|
|
23
|
+
child.stdout.on("data", (chunk) => {
|
|
24
|
+
stdout += chunk.toString("utf8");
|
|
25
|
+
});
|
|
26
|
+
child.stderr.on("data", (chunk) => {
|
|
27
|
+
stderr += chunk.toString("utf8");
|
|
28
|
+
});
|
|
29
|
+
child.on("close", (code) => {
|
|
30
|
+
clearTimeout(timer);
|
|
31
|
+
const exitCode = timedOut ? 124 : (code ?? 1);
|
|
32
|
+
if (timedOut) {
|
|
33
|
+
stderr += `\nCommand timed out after ${timeoutMs}ms`;
|
|
34
|
+
}
|
|
35
|
+
const result = compressOutput({ command, stdout, stderr, exitCode });
|
|
36
|
+
if (saveRaw) {
|
|
37
|
+
const runId = generateRunId();
|
|
38
|
+
const combinedClean = stripAnsi([stdout, stderr].filter(Boolean).join("\n"));
|
|
39
|
+
saveRun({
|
|
40
|
+
runId,
|
|
41
|
+
command,
|
|
42
|
+
cwd,
|
|
43
|
+
exitCode,
|
|
44
|
+
parser: result.parser,
|
|
45
|
+
timestamp: new Date().toISOString(),
|
|
46
|
+
rawTokens: result.rawTokens,
|
|
47
|
+
compressedTokens: result.compressedTokens,
|
|
48
|
+
rawLines: result.rawLines,
|
|
49
|
+
rawOutput: combinedClean,
|
|
50
|
+
});
|
|
51
|
+
result.runId = runId;
|
|
52
|
+
}
|
|
53
|
+
resolve(result);
|
|
54
|
+
});
|
|
55
|
+
child.on("error", (err) => {
|
|
56
|
+
clearTimeout(timer);
|
|
57
|
+
const result = compressOutput({
|
|
58
|
+
command,
|
|
59
|
+
stdout,
|
|
60
|
+
stderr: `${stderr}\n${err.message}`.trim(),
|
|
61
|
+
exitCode: 1,
|
|
62
|
+
});
|
|
63
|
+
resolve(result);
|
|
64
|
+
});
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Formats a result for the agent. The footer is kept short because it is paid
|
|
69
|
+
* on every call: run id (for `skim inspect`), raw size, and exit code.
|
|
70
|
+
* Token stats are left out of the footer; they are in `skim stats`.
|
|
71
|
+
*/
|
|
72
|
+
export function formatSkimOutput(result) {
|
|
73
|
+
if (result.passthrough) {
|
|
74
|
+
return result.exitCode !== 0
|
|
75
|
+
? `${result.summary}\n[exit ${result.exitCode}]`
|
|
76
|
+
: result.summary;
|
|
77
|
+
}
|
|
78
|
+
const parts = [result.runId ? `skim ${result.runId}` : "skim", `${result.rawLines} lines`];
|
|
79
|
+
if (result.exitCode !== 0)
|
|
80
|
+
parts.push(`exit ${result.exitCode}`);
|
|
81
|
+
return `${result.summary}\n[${parts.join(" · ")}]`;
|
|
82
|
+
}
|
|
83
|
+
//# sourceMappingURL=runner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/core/runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAUxC,MAAM,UAAU,aAAa,CAC3B,OAAe,EACf,UAA6B,EAAE;IAE/B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC;IAC/C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC;IAExC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE;YAC3B,GAAG;YACH,KAAK,EAAE,IAAI;YACX,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE;YACvC,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;SAClC,CAAC,CAAC;QAEH,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,QAAQ,GAAG,IAAI,CAAC;YAChB,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACxB,CAAC,EAAE,SAAS,CAAC,CAAC;QAEd,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;YACxC,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;QAEH,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;YACxC,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;QAEH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;YACzB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,QAAQ,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;YAC9C,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,IAAI,6BAA6B,SAAS,IAAI,CAAC;YACvD,CAAC;YAED,MAAM,MAAM,GAAG,cAAc,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;YAErE,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,KAAK,GAAG,aAAa,EAAE,CAAC;gBAC9B,MAAM,aAAa,GAAG,SAAS,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;gBAC7E,OAAO,CAAC;oBACN,KAAK;oBACL,OAAO;oBACP,GAAG;oBACH,QAAQ;oBACR,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;oBACnC,SAAS,EAAE,MAAM,CAAC,SAAS;oBAC3B,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;oBACzC,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,SAAS,EAAE,aAAa;iBACzB,CAAC,CAAC;gBACH,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;YACvB,CAAC;YAED,OAAO,CAAC,MAAM,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;QAEH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACxB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,MAAM,GAAG,cAAc,CAAC;gBAC5B,OAAO;gBACP,MAAM;gBACN,MAAM,EAAE,GAAG,MAAM,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE;gBAC1C,QAAQ,EAAE,CAAC;aACZ,CAAC,CAAC;YACH,OAAO,CAAC,MAAM,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAsB;IACrD,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QACvB,OAAO,MAAM,CAAC,QAAQ,KAAK,CAAC;YAC1B,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,WAAW,MAAM,CAAC,QAAQ,GAAG;YAChD,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;IACrB,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,QAAQ,CAAC,CAAC;IAC3F,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IACjE,OAAO,GAAG,MAAM,CAAC,OAAO,MAAM,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;AACrD,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { InspectOptions, InspectResult, RunRecord, SkimStats } from "./types.js";
|
|
2
|
+
export declare function getSkimDir(): string;
|
|
3
|
+
export declare function generateRunId(): string;
|
|
4
|
+
export declare function saveRun(record: RunRecord): void;
|
|
5
|
+
export declare function loadRun(runId: string): RunRecord | null;
|
|
6
|
+
export declare function inspectRun(runId: string, options?: InspectOptions): InspectResult | null;
|
|
7
|
+
export declare function loadStats(): SkimStats;
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
import crypto from "node:crypto";
|
|
2
|
+
import fs from "node:fs";
|
|
3
|
+
import os from "node:os";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
const MAX_STORED_RUNS = 100;
|
|
6
|
+
export function getSkimDir() {
|
|
7
|
+
return process.env.SKIM_DIR || path.join(os.homedir(), ".skim");
|
|
8
|
+
}
|
|
9
|
+
function getRunsDir() {
|
|
10
|
+
const dir = path.join(getSkimDir(), "runs");
|
|
11
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
12
|
+
return dir;
|
|
13
|
+
}
|
|
14
|
+
function getStatsPath() {
|
|
15
|
+
const dir = getSkimDir();
|
|
16
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
17
|
+
return path.join(dir, "stats.json");
|
|
18
|
+
}
|
|
19
|
+
export function generateRunId() {
|
|
20
|
+
return `run_${crypto.randomBytes(3).toString("hex")}`;
|
|
21
|
+
}
|
|
22
|
+
export function saveRun(record) {
|
|
23
|
+
const runsDir = getRunsDir();
|
|
24
|
+
const filePath = path.join(runsDir, `${record.runId}.json`);
|
|
25
|
+
fs.writeFileSync(filePath, JSON.stringify(record), "utf8");
|
|
26
|
+
pruneOldRuns(runsDir);
|
|
27
|
+
recordStats(record);
|
|
28
|
+
}
|
|
29
|
+
export function loadRun(runId) {
|
|
30
|
+
const safeId = path.basename(runId);
|
|
31
|
+
const filePath = path.join(getRunsDir(), `${safeId}.json`);
|
|
32
|
+
if (!fs.existsSync(filePath))
|
|
33
|
+
return null;
|
|
34
|
+
try {
|
|
35
|
+
return JSON.parse(fs.readFileSync(filePath, "utf8"));
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
return null;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
export function inspectRun(runId, options = {}) {
|
|
42
|
+
const record = loadRun(runId);
|
|
43
|
+
if (!record)
|
|
44
|
+
return null;
|
|
45
|
+
const lines = record.rawOutput.split("\n");
|
|
46
|
+
const totalLines = lines.length;
|
|
47
|
+
const maxLines = Math.max(1, Math.min(options.maxLines ?? 80, 300));
|
|
48
|
+
if (options.query) {
|
|
49
|
+
const context = Math.max(0, Math.min(options.contextLines ?? 2, 10));
|
|
50
|
+
const q = options.query.toLowerCase();
|
|
51
|
+
const matchedIndices = new Set();
|
|
52
|
+
for (let i = 0; i < lines.length; i++) {
|
|
53
|
+
if (lines[i].toLowerCase().includes(q)) {
|
|
54
|
+
const start = Math.max(0, i - context);
|
|
55
|
+
const end = Math.min(lines.length - 1, i + context);
|
|
56
|
+
for (let j = start; j <= end; j++) {
|
|
57
|
+
matchedIndices.add(j);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
const sorted = Array.from(matchedIndices).sort((a, b) => a - b);
|
|
62
|
+
const sliced = sorted.slice(0, maxLines);
|
|
63
|
+
const out = [];
|
|
64
|
+
let prev = -2;
|
|
65
|
+
for (const idx of sliced) {
|
|
66
|
+
if (prev >= 0 && idx > prev + 1) {
|
|
67
|
+
out.push("...");
|
|
68
|
+
}
|
|
69
|
+
out.push(`${idx + 1}: ${lines[idx]}`);
|
|
70
|
+
prev = idx;
|
|
71
|
+
}
|
|
72
|
+
if (sorted.length > maxLines) {
|
|
73
|
+
out.push(`... (${sorted.length - maxLines} more matching lines omitted)`);
|
|
74
|
+
}
|
|
75
|
+
return {
|
|
76
|
+
runId: record.runId,
|
|
77
|
+
command: record.command,
|
|
78
|
+
totalLines,
|
|
79
|
+
returnedLines: sliced.length,
|
|
80
|
+
content: out.length > 0 ? out.join("\n") : `No lines matching "${options.query}" in ${runId}.`,
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
const startLine = Math.max(1, options.startLine ?? 1);
|
|
84
|
+
const endLine = Math.min(totalLines, options.endLine ?? startLine + maxLines - 1);
|
|
85
|
+
const clampedEnd = Math.min(endLine, startLine + maxLines - 1);
|
|
86
|
+
const out = [];
|
|
87
|
+
for (let i = startLine - 1; i < clampedEnd; i++) {
|
|
88
|
+
out.push(`${i + 1}: ${lines[i]}`);
|
|
89
|
+
}
|
|
90
|
+
if (clampedEnd < totalLines) {
|
|
91
|
+
out.push(`... (${totalLines - clampedEnd} more lines; use startLine=${clampedEnd + 1} to continue)`);
|
|
92
|
+
}
|
|
93
|
+
return {
|
|
94
|
+
runId: record.runId,
|
|
95
|
+
command: record.command,
|
|
96
|
+
totalLines,
|
|
97
|
+
returnedLines: clampedEnd - startLine + 1,
|
|
98
|
+
content: out.join("\n"),
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
export function loadStats() {
|
|
102
|
+
const statsPath = getStatsPath();
|
|
103
|
+
if (!fs.existsSync(statsPath)) {
|
|
104
|
+
return {
|
|
105
|
+
totalRuns: 0,
|
|
106
|
+
totalRawTokens: 0,
|
|
107
|
+
totalCompressedTokens: 0,
|
|
108
|
+
totalSavedTokens: 0,
|
|
109
|
+
byParser: {},
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
try {
|
|
113
|
+
return JSON.parse(fs.readFileSync(statsPath, "utf8"));
|
|
114
|
+
}
|
|
115
|
+
catch {
|
|
116
|
+
return {
|
|
117
|
+
totalRuns: 0,
|
|
118
|
+
totalRawTokens: 0,
|
|
119
|
+
totalCompressedTokens: 0,
|
|
120
|
+
totalSavedTokens: 0,
|
|
121
|
+
byParser: {},
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
function recordStats(record) {
|
|
126
|
+
const stats = loadStats();
|
|
127
|
+
const saved = Math.max(0, record.rawTokens - record.compressedTokens);
|
|
128
|
+
stats.totalRuns += 1;
|
|
129
|
+
stats.totalRawTokens += record.rawTokens;
|
|
130
|
+
stats.totalCompressedTokens += record.compressedTokens;
|
|
131
|
+
stats.totalSavedTokens += saved;
|
|
132
|
+
if (!stats.byParser[record.parser]) {
|
|
133
|
+
stats.byParser[record.parser] = { runs: 0, rawTokens: 0, compressedTokens: 0 };
|
|
134
|
+
}
|
|
135
|
+
stats.byParser[record.parser].runs += 1;
|
|
136
|
+
stats.byParser[record.parser].rawTokens += record.rawTokens;
|
|
137
|
+
stats.byParser[record.parser].compressedTokens += record.compressedTokens;
|
|
138
|
+
fs.writeFileSync(getStatsPath(), JSON.stringify(stats, null, 2), "utf8");
|
|
139
|
+
}
|
|
140
|
+
function pruneOldRuns(runsDir) {
|
|
141
|
+
const files = fs
|
|
142
|
+
.readdirSync(runsDir)
|
|
143
|
+
.filter((f) => f.endsWith(".json"))
|
|
144
|
+
.map((f) => {
|
|
145
|
+
const full = path.join(runsDir, f);
|
|
146
|
+
return { full, mtime: fs.statSync(full).mtimeMs };
|
|
147
|
+
})
|
|
148
|
+
.sort((a, b) => b.mtime - a.mtime);
|
|
149
|
+
for (let i = MAX_STORED_RUNS; i < files.length; i++) {
|
|
150
|
+
try {
|
|
151
|
+
fs.unlinkSync(files[i].full);
|
|
152
|
+
}
|
|
153
|
+
catch {
|
|
154
|
+
// ignore race conditions
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
//# sourceMappingURL=store.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/core/store.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B,MAAM,UAAU,UAAU;IACxB,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,OAAO,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,UAAU;IACjB,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,MAAM,CAAC,CAAC;IAC5C,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACvC,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,YAAY;IACnB,MAAM,GAAG,GAAG,UAAU,EAAE,CAAC;IACzB,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACvC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,OAAO,OAAO,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;AACxD,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,MAAiB;IACvC,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC;IAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,KAAK,OAAO,CAAC,CAAC;IAC5D,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;IAC3D,YAAY,CAAC,OAAO,CAAC,CAAC;IACtB,WAAW,CAAC,MAAM,CAAC,CAAC;AACtB,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,KAAa;IACnC,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,GAAG,MAAM,OAAO,CAAC,CAAC;IAC3D,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1C,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAc,CAAC;IACpE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,KAAa,EAAE,UAA0B,EAAE;IACpE,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;IAC9B,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAEzB,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC;IAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;IAEpE,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAClB,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,YAAY,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QACrE,MAAM,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;QACtC,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;QAEzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC;gBACvC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC;gBACpD,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;oBAClC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBACxB,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAChE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QACzC,MAAM,GAAG,GAAa,EAAE,CAAC;QACzB,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC;QACd,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;YACzB,IAAI,IAAI,IAAI,CAAC,IAAI,GAAG,GAAG,IAAI,GAAG,CAAC,EAAE,CAAC;gBAChC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClB,CAAC;YACD,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,KAAK,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACtC,IAAI,GAAG,GAAG,CAAC;QACb,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YAC7B,GAAG,CAAC,IAAI,CAAC,QAAQ,MAAM,CAAC,MAAM,GAAG,QAAQ,+BAA+B,CAAC,CAAC;QAC5E,CAAC;QAED,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,UAAU;YACV,aAAa,EAAE,MAAM,CAAC,MAAM;YAC5B,OAAO,EAAE,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,sBAAsB,OAAO,CAAC,KAAK,QAAQ,KAAK,GAAG;SAC/F,CAAC;IACJ,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC;IACtD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,OAAO,CAAC,OAAO,IAAI,SAAS,GAAG,QAAQ,GAAG,CAAC,CAAC,CAAC;IAClF,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,SAAS,GAAG,QAAQ,GAAG,CAAC,CAAC,CAAC;IAE/D,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,IAAI,CAAC,GAAG,SAAS,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;QAChD,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACpC,CAAC;IACD,IAAI,UAAU,GAAG,UAAU,EAAE,CAAC;QAC5B,GAAG,CAAC,IAAI,CAAC,QAAQ,UAAU,GAAG,UAAU,8BAA8B,UAAU,GAAG,CAAC,eAAe,CAAC,CAAC;IACvG,CAAC;IAED,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,UAAU;QACV,aAAa,EAAE,UAAU,GAAG,SAAS,GAAG,CAAC;QACzC,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;KACxB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS;IACvB,MAAM,SAAS,GAAG,YAAY,EAAE,CAAC;IACjC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9B,OAAO;YACL,SAAS,EAAE,CAAC;YACZ,cAAc,EAAE,CAAC;YACjB,qBAAqB,EAAE,CAAC;YACxB,gBAAgB,EAAE,CAAC;YACnB,QAAQ,EAAE,EAAE;SACb,CAAC;IACJ,CAAC;IACD,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,MAAM,CAAC,CAAc,CAAC;IACrE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;YACL,SAAS,EAAE,CAAC;YACZ,cAAc,EAAE,CAAC;YACjB,qBAAqB,EAAE,CAAC;YACxB,gBAAgB,EAAE,CAAC;YACnB,QAAQ,EAAE,EAAE;SACb,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,MAAiB;IACpC,MAAM,KAAK,GAAG,SAAS,EAAE,CAAC;IAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,SAAS,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;IACtE,KAAK,CAAC,SAAS,IAAI,CAAC,CAAC;IACrB,KAAK,CAAC,cAAc,IAAI,MAAM,CAAC,SAAS,CAAC;IACzC,KAAK,CAAC,qBAAqB,IAAI,MAAM,CAAC,gBAAgB,CAAC;IACvD,KAAK,CAAC,gBAAgB,IAAI,KAAK,CAAC;IAEhC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;QACnC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC;IACjF,CAAC;IACD,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;IACxC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;IAC5D,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,gBAAgB,IAAI,MAAM,CAAC,gBAAgB,CAAC;IAE1E,EAAE,CAAC,aAAa,CAAC,YAAY,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;AAC3E,CAAC;AAED,SAAS,YAAY,CAAC,OAAe;IACnC,MAAM,KAAK,GAAG,EAAE;SACb,WAAW,CAAC,OAAO,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;SAClC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACnC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;IACpD,CAAC,CAAC;SACD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAErC,KAAK,IAAI,CAAC,GAAG,eAAe,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpD,IAAI,CAAC;YACH,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,yBAAyB;QAC3B,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export declare function stripAnsi(text: string): string;
|
|
2
|
+
export declare function countTokens(text: string): number;
|
|
3
|
+
/** @deprecated Use countTokens. Kept as an alias so existing SDK callers keep working. */
|
|
4
|
+
export declare const estimateTokens: typeof countTokens;
|
|
5
|
+
export declare function countLines(text: string): number;
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { Tiktoken } from "js-tiktoken/lite";
|
|
2
|
+
import o200k_base from "js-tiktoken/ranks/o200k_base";
|
|
3
|
+
// Strip ANSI CSI/OSC escape sequences and collapse carriage-return progress updates
|
|
4
|
+
const ANSI_REGEX =
|
|
5
|
+
// eslint-disable-next-line no-control-regex
|
|
6
|
+
/[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><~]|\u001b\][^\u0007\u001b]*(?:\u0007|\u001b\\)/g;
|
|
7
|
+
export function stripAnsi(text) {
|
|
8
|
+
const noAnsi = text.replace(ANSI_REGEX, "");
|
|
9
|
+
// Collapse carriage-return progress overwrites: keep only the final segment on each line
|
|
10
|
+
return noAnsi
|
|
11
|
+
.replace(/\r\n/g, "\n")
|
|
12
|
+
.split("\n")
|
|
13
|
+
.map((line) => {
|
|
14
|
+
if (!line.includes("\r"))
|
|
15
|
+
return line;
|
|
16
|
+
const parts = line.split("\r").filter((p) => p.length > 0);
|
|
17
|
+
return parts.length > 0 ? parts[parts.length - 1] : "";
|
|
18
|
+
})
|
|
19
|
+
.join("\n");
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Real token count using the o200k_base BPE tokenizer (GPT-4o family).
|
|
23
|
+
* Exact for OpenAI models and a close proxy for Claude and most modern LLMs.
|
|
24
|
+
* The encoder is created lazily on first use, since loading the ranks takes ~100ms.
|
|
25
|
+
*/
|
|
26
|
+
let encoder = null;
|
|
27
|
+
export function countTokens(text) {
|
|
28
|
+
if (!text)
|
|
29
|
+
return 0;
|
|
30
|
+
if (!encoder)
|
|
31
|
+
encoder = new Tiktoken(o200k_base);
|
|
32
|
+
return encoder.encode(text).length;
|
|
33
|
+
}
|
|
34
|
+
/** @deprecated Use countTokens. Kept as an alias so existing SDK callers keep working. */
|
|
35
|
+
export const estimateTokens = countTokens;
|
|
36
|
+
export function countLines(text) {
|
|
37
|
+
if (!text)
|
|
38
|
+
return 0;
|
|
39
|
+
return text.split("\n").length;
|
|
40
|
+
}
|
|
41
|
+
//# sourceMappingURL=tokens.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tokens.js","sourceRoot":"","sources":["../../src/core/tokens.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,UAAU,MAAM,8BAA8B,CAAC;AAEtD,oFAAoF;AACpF,MAAM,UAAU;AACd,4CAA4C;AAC5C,0HAA0H,CAAC;AAE7H,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAC5C,yFAAyF;IACzF,OAAO,MAAM;SACV,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC;SACtB,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACZ,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QACtC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC3D,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACzD,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,IAAI,OAAO,GAAoB,IAAI,CAAC;AAEpC,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IACpB,IAAI,CAAC,OAAO;QAAE,OAAO,GAAG,IAAI,QAAQ,CAAC,UAAU,CAAC,CAAC;IACjD,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;AACrC,CAAC;AAED,0FAA0F;AAC1F,MAAM,CAAC,MAAM,cAAc,GAAG,WAAW,CAAC;AAE1C,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IACpB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;AACjC,CAAC"}
|