@webywz/context-diet 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -17
- package/dist/cli.d.ts +1 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +18 -7
- package/dist/cli.js.map +1 -1
- package/dist/diet.d.ts.map +1 -1
- package/dist/diet.js +14 -2
- package/dist/diet.js.map +1 -1
- package/dist/heuristics.d.ts +15 -1
- package/dist/heuristics.d.ts.map +1 -1
- package/dist/heuristics.js +95 -1
- package/dist/heuristics.js.map +1 -1
- package/dist/html.d.ts +7 -0
- package/dist/html.d.ts.map +1 -0
- package/dist/html.js +62 -0
- package/dist/html.js.map +1 -0
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/parse-args.d.ts +5 -1
- package/dist/parse-args.d.ts.map +1 -1
- package/dist/parse-args.js +30 -1
- package/dist/parse-args.js.map +1 -1
- package/dist/presets.d.ts +29 -0
- package/dist/presets.d.ts.map +1 -0
- package/dist/presets.js +50 -0
- package/dist/presets.js.map +1 -0
- package/dist/report.d.ts +14 -1
- package/dist/report.d.ts.map +1 -1
- package/dist/report.js +65 -8
- package/dist/report.js.map +1 -1
- package/dist/truncate.d.ts +1 -1
- package/dist/truncate.d.ts.map +1 -1
- package/dist/truncate.js +51 -5
- package/dist/truncate.js.map +1 -1
- package/dist/types.d.ts +14 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +3 -0
- package/dist/types.js.map +1 -1
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -9,7 +9,7 @@ npm i -g @webywz/context-diet
|
|
|
9
9
|
```
|
|
10
10
|
|
|
11
11
|
```bash
|
|
12
|
-
context-diet logs.txt --max-tokens 2000
|
|
12
|
+
context-diet logs.txt --preset logs --max-tokens 2000
|
|
13
13
|
```
|
|
14
14
|
|
|
15
15
|
## Install
|
|
@@ -32,15 +32,29 @@ npx @webywz/context-diet --help
|
|
|
32
32
|
context-diet input.txt
|
|
33
33
|
|
|
34
34
|
# stdin
|
|
35
|
-
cat noisy.log | context-diet --max-tokens 1500 > slim.txt
|
|
35
|
+
cat noisy.log | context-diet --preset logs --max-tokens 1500 > slim.txt
|
|
36
36
|
|
|
37
37
|
# write the body to a file
|
|
38
38
|
context-diet notes.md -o slim.md --keep headings,code,lists
|
|
39
39
|
|
|
40
|
+
# report only (no body / no -o write)
|
|
41
|
+
context-diet input.txt --dry-run --diff
|
|
42
|
+
|
|
40
43
|
# machine-readable
|
|
41
44
|
context-diet input.txt --json | jq '{tokens: .output.tokens, cuts}'
|
|
42
45
|
```
|
|
43
46
|
|
|
47
|
+
### Presets
|
|
48
|
+
|
|
49
|
+
| Preset | Keeps by default | Extra strippers |
|
|
50
|
+
| --- | --- | --- |
|
|
51
|
+
| `logs` | `code` | Prefer `ERROR`/`WARN`/`FATAL` when truncating |
|
|
52
|
+
| `email` | `headings`, `lists` | Quoted replies (`>` / `On … wrote:` / Original Message) |
|
|
53
|
+
| `chat` | `code`, `lists` | Reactions, join/leave system lines, lone timestamps |
|
|
54
|
+
| `html` | `headings`, `code`, `lists` | `<script>` / `<style>` / comments / tags → prose |
|
|
55
|
+
|
|
56
|
+
Explicit `--keep` always wins over the preset default.
|
|
57
|
+
|
|
44
58
|
### Before / after
|
|
45
59
|
|
|
46
60
|
Input (`examples/noisy.txt`) — repeated logs, a duplicated runbook, a signature, a fat blob:
|
|
@@ -48,19 +62,11 @@ Input (`examples/noisy.txt`) — repeated logs, a duplicated runbook, a signatur
|
|
|
48
62
|
```
|
|
49
63
|
[2026-03-14 09:01:02] INFO Starting request handler
|
|
50
64
|
[2026-03-14 09:01:02] DEBUG loading config from /etc/app/config.yaml
|
|
51
|
-
[2026-03-14 09:01:03] DEBUG loading config from /etc/app/config.yaml
|
|
52
|
-
[2026-03-14 09:01:03] DEBUG loading config from /etc/app/config.yaml
|
|
53
65
|
…
|
|
54
66
|
CONFIDENTIALITY NOTICE
|
|
55
|
-
This email is confidential and intended only for the recipient.
|
|
56
67
|
…
|
|
57
|
-
|
|
58
|
-
The deploy runbook covers staging and production.
|
|
59
|
-
The deploy runbook covers staging and production rollouts.
|
|
60
|
-
…
|
|
61
|
-
|
|
62
68
|
Sent from my iPhone
|
|
63
|
-
data:image/png;base64,
|
|
69
|
+
data:image/png;base64,iVBORw0KGgo… (huge)
|
|
64
70
|
```
|
|
65
71
|
|
|
66
72
|
```bash
|
|
@@ -70,12 +76,15 @@ context-diet examples/noisy.txt --keep headings|code|lists
|
|
|
70
76
|
Stderr:
|
|
71
77
|
|
|
72
78
|
```
|
|
73
|
-
context-diet 0.
|
|
79
|
+
context-diet 0.2.0
|
|
74
80
|
source: examples/noisy.txt
|
|
75
81
|
chars: 1,666 → 834 (−49.9%)
|
|
76
82
|
tokens: ~667 → ~312 (−53.2%)
|
|
77
83
|
cut:
|
|
78
84
|
ansi sequences 0
|
|
85
|
+
html chrome 0
|
|
86
|
+
quoted replies 0
|
|
87
|
+
chat chrome 0
|
|
79
88
|
boilerplate lines 9
|
|
80
89
|
duplicate paragraphs 2
|
|
81
90
|
whitespace runs 5
|
|
@@ -94,11 +103,14 @@ All of this is **offline heuristics**. No network unless you pass `--deepen`.
|
|
|
94
103
|
| Step | Behavior |
|
|
95
104
|
| --- | --- |
|
|
96
105
|
| Strip ANSI | Terminal color / OSC sequences |
|
|
106
|
+
| Strip HTML | Optional / `--preset html`: scripts, styles, comments, tags |
|
|
97
107
|
| Omit heavy blobs | Long base64 / data-URIs → `[omitted blob: N chars]` |
|
|
98
108
|
| Collapse whitespace | Trailing space, squeezed blanks — **never inside code fences** |
|
|
99
109
|
| Drop boilerplate | Repeated log lines (timestamps/UUIDs normalized), signatures, `DEBUG` chrome, horizontal rules |
|
|
100
|
-
|
|
|
101
|
-
|
|
|
110
|
+
| Drop quoted replies | `--preset email`: `>` quotes, `On … wrote:`, Original Message tails |
|
|
111
|
+
| Drop chat chrome | `--preset chat`: reactions, join/leave, lone timestamps |
|
|
112
|
+
| Dedupe paragraphs | Exact and near-identical (Jaccard ≥ 0.84) prose/list blocks |
|
|
113
|
+
| Truncate | `--max-tokens N` drops low-value blocks first; `--keep` / presets pin types; `logs` also prefers error lines |
|
|
102
114
|
|
|
103
115
|
Token estimates are **cl100k-ish** (GPT-4 family grain: ~4 chars/word, 1 CJK char ≈ 1 token, punctuation is dense). Good enough to budget a prompt. Not a billed tokenizer — do not invoice with it.
|
|
104
116
|
|
|
@@ -114,7 +126,10 @@ context-diet [file] [options]
|
|
|
114
126
|
| `-o, --output <file>` | Write slimmed text here instead of stdout |
|
|
115
127
|
| `--max-tokens <n>` | Cap estimated output tokens |
|
|
116
128
|
| `--keep <cats>` | Prefer `headings`, `code`, `lists` when truncating (comma or `\|`) |
|
|
117
|
-
| `--
|
|
129
|
+
| `--preset <name>` | `logs` \| `email` \| `chat` \| `html` |
|
|
130
|
+
| `--json` | JSON report on stdout (includes `text`, unless `-o` / `--dry-run`) |
|
|
131
|
+
| `--dry-run` | Report only; do not emit slimmed body or write `-o` |
|
|
132
|
+
| `--diff` | Show removed lines on stderr (capped) |
|
|
118
133
|
| `--deepen` | Optional LLM rewrite (see below) |
|
|
119
134
|
| `--provider openai\|anthropic` | Force a provider for `--deepen` |
|
|
120
135
|
| `-q, --quiet` | No human report on stderr |
|
|
@@ -124,9 +139,11 @@ context-diet [file] [options]
|
|
|
124
139
|
Default I/O: **slimmed body on stdout**, **human report on stderr**, so this stays pipe-friendly:
|
|
125
140
|
|
|
126
141
|
```bash
|
|
127
|
-
pbpaste | context-diet --max-tokens 800
|
|
142
|
+
pbpaste | context-diet --preset chat --max-tokens 800 | pbcopy
|
|
128
143
|
```
|
|
129
144
|
|
|
145
|
+
The human report respects `NO_COLOR` / `FORCE_COLOR`.
|
|
146
|
+
|
|
130
147
|
## Optional LLM pass
|
|
131
148
|
|
|
132
149
|
`--deepen` is **never required**. If you already have a key and want a second squeeze after the local diet:
|
|
@@ -145,7 +162,8 @@ import { diet, dietAsync, estimateTokens } from "@webywz/context-diet";
|
|
|
145
162
|
|
|
146
163
|
const { text, input, output, cuts } = diet(raw, {
|
|
147
164
|
maxTokens: 2000,
|
|
148
|
-
|
|
165
|
+
preset: "logs",
|
|
166
|
+
keep: ["code"], // optional override
|
|
149
167
|
});
|
|
150
168
|
|
|
151
169
|
await dietAsync(raw, { deepen: true }); // keys from process.env
|
package/dist/cli.d.ts
CHANGED
|
@@ -4,6 +4,7 @@ export interface Io {
|
|
|
4
4
|
stdout: NodeJS.WritableStream;
|
|
5
5
|
stderr: NodeJS.WritableStream;
|
|
6
6
|
stdinIsTTY: boolean | undefined;
|
|
7
|
+
stderrIsTTY?: boolean | undefined;
|
|
7
8
|
env: NodeJS.ProcessEnv;
|
|
8
9
|
readFile: (path: string) => Promise<string>;
|
|
9
10
|
writeFile: (path: string, contents: string) => Promise<void>;
|
package/dist/cli.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";
|
|
1
|
+
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAgBA,MAAM,WAAW,EAAE;IACjB,KAAK,EAAE,MAAM,CAAC,cAAc,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC,cAAc,CAAC;IAC9B,MAAM,EAAE,MAAM,CAAC,cAAc,CAAC;IAC9B,UAAU,EAAE,OAAO,GAAG,SAAS,CAAC;IAChC,WAAW,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAClC,GAAG,EAAE,MAAM,CAAC,UAAU,CAAC;IACvB,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IAC5C,SAAS,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9D;AAsDD,wBAAsB,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,GAAE,EAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,CAmF/E"}
|
package/dist/cli.js
CHANGED
|
@@ -7,12 +7,13 @@ import { dietAsync } from "./diet.js";
|
|
|
7
7
|
import { DeepenError } from "./llm.js";
|
|
8
8
|
import { packageInfo } from "./package-info.js";
|
|
9
9
|
import { HELP_TEXT, parseArgs, UsageError } from "./parse-args.js";
|
|
10
|
-
import { formatHumanReport, toJsonReport } from "./report.js";
|
|
10
|
+
import { formatHumanReport, formatRemovedDiff, shouldUseColor, toJsonReport, } from "./report.js";
|
|
11
11
|
const defaultIo = () => ({
|
|
12
12
|
stdin: process.stdin,
|
|
13
13
|
stdout: process.stdout,
|
|
14
14
|
stderr: process.stderr,
|
|
15
15
|
stdinIsTTY: process.stdin.isTTY,
|
|
16
|
+
stderrIsTTY: process.stderr.isTTY,
|
|
16
17
|
env: process.env,
|
|
17
18
|
readFile: (path) => readFile(path, "utf8"),
|
|
18
19
|
writeFile: (path, contents) => writeFile(path, contents, "utf8"),
|
|
@@ -95,6 +96,7 @@ export async function run(argv, io = defaultIo()) {
|
|
|
95
96
|
result = await dietAsync(text, {
|
|
96
97
|
maxTokens: args.maxTokens,
|
|
97
98
|
keep: args.keep,
|
|
99
|
+
preset: args.preset,
|
|
98
100
|
deepen: args.deepen,
|
|
99
101
|
provider: args.provider,
|
|
100
102
|
env: io.env,
|
|
@@ -108,19 +110,28 @@ export async function run(argv, io = defaultIo()) {
|
|
|
108
110
|
throw err;
|
|
109
111
|
}
|
|
110
112
|
const json = toJsonReport(result, source);
|
|
111
|
-
|
|
113
|
+
const color = shouldUseColor(io.env, io.stderrIsTTY);
|
|
114
|
+
const style = { color };
|
|
115
|
+
if (args.output && !args.dryRun) {
|
|
112
116
|
await io.writeFile(args.output, result.text);
|
|
113
117
|
}
|
|
114
118
|
if (args.json) {
|
|
115
|
-
const payload = args.
|
|
119
|
+
const payload = args.dryRun
|
|
120
|
+
? { ...json, text: undefined, dryRun: true }
|
|
121
|
+
: args.output
|
|
122
|
+
? { ...json, text: undefined, outputPath: args.output }
|
|
123
|
+
: json;
|
|
116
124
|
await writeOut(io.stdout, `${JSON.stringify(payload, null, 2)}\n`);
|
|
117
|
-
return 0;
|
|
118
125
|
}
|
|
119
|
-
if (!args.output) {
|
|
126
|
+
else if (!args.dryRun && !args.output) {
|
|
120
127
|
await writeOut(io.stdout, result.text);
|
|
121
128
|
}
|
|
122
|
-
|
|
123
|
-
|
|
129
|
+
// --json is the machine report; skip the human summary (unless quiet already).
|
|
130
|
+
if (!args.json && !args.quiet) {
|
|
131
|
+
await writeOut(io.stderr, formatHumanReport(result, source, version, style));
|
|
132
|
+
}
|
|
133
|
+
if (args.diff) {
|
|
134
|
+
await writeOut(io.stderr, formatRemovedDiff(text, result.text, style));
|
|
124
135
|
}
|
|
125
136
|
return 0;
|
|
126
137
|
}
|
package/dist/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,OAAO,MAAM,cAAc,CAAC;AACnC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAgB,MAAM,iBAAiB,CAAC;AACjF,OAAO,
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,OAAO,MAAM,cAAc,CAAC;AACnC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAgB,MAAM,iBAAiB,CAAC;AACjF,OAAO,EACL,iBAAiB,EACjB,iBAAiB,EACjB,cAAc,EACd,YAAY,GACb,MAAM,aAAa,CAAC;AAarB,MAAM,SAAS,GAAG,GAAO,EAAE,CAAC,CAAC;IAC3B,KAAK,EAAE,OAAO,CAAC,KAAK;IACpB,MAAM,EAAE,OAAO,CAAC,MAAM;IACtB,MAAM,EAAE,OAAO,CAAC,MAAM;IACtB,UAAU,EAAE,OAAO,CAAC,KAAK,CAAC,KAAK;IAC/B,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK;IACjC,GAAG,EAAE,OAAO,CAAC,GAAG;IAChB,QAAQ,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAC1C,SAAS,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC;CACjE,CAAC,CAAC;AAEH,SAAS,UAAU,CAAC,MAA6B;IAC/C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAkB,EAAE,EAAE;YACvC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACxE,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,IAAa,EAAE,EAAM;IAC5C,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,GAAG,EAAE,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1C,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;QACrC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,MAAM,IAAI,UAAU,CAAC,eAAe,IAAI,CAAC,IAAI,KAAK,GAAG,EAAE,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,GAAG,CAAC;IACjD,IAAI,QAAQ,IAAI,EAAE,CAAC,UAAU,EAAE,CAAC;QAC9B,MAAM,IAAI,UAAU,CAClB,2EAA2E,CAC5E,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC;IACxC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;AACnC,CAAC;AAED,SAAS,QAAQ,CAAC,MAA6B,EAAE,KAAa;IAC5D,OAAO,IAAI,OAAO,CAAC,CAAC,YAAY,EAAE,MAAM,EAAE,EAAE;QAC1C,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE;YAClC,IAAI,GAAG;gBAAE,MAAM,CAAC,GAAG,CAAC,CAAC;;gBAChB,YAAY,EAAE,CAAC;QACtB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,GAAG,CAAC,IAAc,EAAE,KAAS,SAAS,EAAE;IAC5D,IAAI,IAAa,CAAC;IAClB,IAAI,CAAC;QACH,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,GAAG,YAAY,UAAU,EAAE,CAAC;YAC9B,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,OAAO,SAAS,EAAE,CAAC,CAAC;YAC5D,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,GAAG,CAAC;IACZ,CAAC;IAED,MAAM,EAAE,OAAO,EAAE,GAAG,WAAW,EAAE,CAAC;IAElC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QACrC,OAAO,CAAC,CAAC;IACX,CAAC;IACD,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QACjB,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,OAAO,IAAI,CAAC,CAAC;QAC1C,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,MAAc,CAAC;IACnB,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACxC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;QACtB,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;IACpB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,GAAG,YAAY,UAAU,EAAE,CAAC;YAC9B,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC;YAC9C,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,GAAG,CAAC;IACZ,CAAC;IAED,IAAI,MAAM,CAAC;IACX,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE;YAC7B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,GAAG,EAAE,EAAE,CAAC,GAAG;SACZ,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,GAAG,YAAY,WAAW,EAAE,CAAC;YAC/B,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC;YAC9C,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,GAAG,CAAC;IACZ,CAAC;IAED,MAAM,IAAI,GAAG,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,MAAM,KAAK,GAAG,cAAc,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC;IACrD,MAAM,KAAK,GAAG,EAAE,KAAK,EAAE,CAAC;IAExB,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QAChC,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;IAC/C,CAAC;IAED,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM;YACzB,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE;YAC5C,CAAC,CAAC,IAAI,CAAC,MAAM;gBACX,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,EAAE;gBACvD,CAAC,CAAC,IAAI,CAAC;QACX,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IACrE,CAAC;SAAM,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACxC,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;IACzC,CAAC;IAED,+EAA+E;IAC/E,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAC9B,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;IAC/E,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,MAAM,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,iBAAiB,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;IACzE,CAAC;IAED,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,YAAY;IACnB,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC9B,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IACzB,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC5C,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC5D,OAAO,IAAI,KAAK,QAAQ,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,IAAI,YAAY,EAAE,EAAE,CAAC;IACnB,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAC7B,CAAC,IAAI,EAAE,EAAE;QACP,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IAC1B,CAAC,EACD,CAAC,GAAY,EAAE,EAAE;QACf,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC1E,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC;QACjC,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACvB,CAAC,CACF,CAAC;AACJ,CAAC"}
|
package/dist/diet.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"diet.d.ts","sourceRoot":"","sources":["../src/diet.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"diet.d.ts","sourceRoot":"","sources":["../src/diet.ts"],"names":[],"mappings":"AAWA,OAAO,EAEL,KAAK,WAAW,EAChB,KAAK,UAAU,EAChB,MAAM,YAAY,CAAC;AA0CpB;;;GAGG;AACH,wBAAgB,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,WAAgB,GAAG,UAAU,CAOxE;AAED;;;GAGG;AACH,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,WAAgB,GACxB,OAAO,CAAC,UAAU,CAAC,CAsBrB"}
|
package/dist/diet.js
CHANGED
|
@@ -1,18 +1,28 @@
|
|
|
1
1
|
import { joinBlocks, parseBlocks } from "./blocks.js";
|
|
2
2
|
import { omitHeavyBlobs, stripAnsi, tidy, } from "./heuristics.js";
|
|
3
|
+
import { stripHtmlChrome } from "./html.js";
|
|
3
4
|
import { deepenText } from "./llm.js";
|
|
5
|
+
import { resolvePreset } from "./presets.js";
|
|
4
6
|
import { snapshot } from "./tokens.js";
|
|
5
7
|
import { truncateToBudget } from "./truncate.js";
|
|
6
8
|
import { emptyCuts, } from "./types.js";
|
|
7
9
|
function localDiet(text, options) {
|
|
8
10
|
const cuts = emptyCuts();
|
|
9
11
|
const input = snapshot(text);
|
|
12
|
+
const resolved = resolvePreset(options.preset, options.keep);
|
|
13
|
+
const stripHtml = options.stripHtml ?? resolved.stripHtml;
|
|
10
14
|
let next = stripAnsi(text, cuts);
|
|
15
|
+
if (stripHtml) {
|
|
16
|
+
next = stripHtmlChrome(next, cuts);
|
|
17
|
+
}
|
|
11
18
|
next = omitHeavyBlobs(next, cuts);
|
|
12
|
-
next = tidy(next, cuts
|
|
19
|
+
next = tidy(next, cuts, {
|
|
20
|
+
stripQuotedReplies: resolved.stripQuotedReplies,
|
|
21
|
+
stripChatChrome: resolved.stripChatChrome,
|
|
22
|
+
});
|
|
13
23
|
if (options.maxTokens && options.maxTokens > 0) {
|
|
14
24
|
const blocks = parseBlocks(next);
|
|
15
|
-
const trimmed = truncateToBudget(blocks, options.maxTokens,
|
|
25
|
+
const trimmed = truncateToBudget(blocks, options.maxTokens, resolved.keep, cuts, resolved.preferErrors);
|
|
16
26
|
next = joinBlocks(trimmed);
|
|
17
27
|
}
|
|
18
28
|
// Final trailing-newline hygiene.
|
|
@@ -34,6 +44,8 @@ export function diet(text, options = {}) {
|
|
|
34
44
|
return localDiet(text, {
|
|
35
45
|
maxTokens: options.maxTokens,
|
|
36
46
|
keep: options.keep,
|
|
47
|
+
preset: options.preset,
|
|
48
|
+
stripHtml: options.stripHtml,
|
|
37
49
|
});
|
|
38
50
|
}
|
|
39
51
|
/**
|
package/dist/diet.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"diet.js","sourceRoot":"","sources":["../src/diet.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EACL,cAAc,EACd,SAAS,EACT,IAAI,GACL,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EACL,SAAS,GAGV,MAAM,YAAY,CAAC;AAEpB,SAAS,SAAS,CAAC,IAAY,EAAE,OAAoB;IACnD,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC;IACzB,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"diet.js","sourceRoot":"","sources":["../src/diet.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EACL,cAAc,EACd,SAAS,EACT,IAAI,GACL,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EACL,SAAS,GAGV,MAAM,YAAY,CAAC;AAEpB,SAAS,SAAS,CAAC,IAAY,EAAE,OAAoB;IACnD,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC;IACzB,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,QAAQ,GAAG,aAAa,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7D,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,QAAQ,CAAC,SAAS,CAAC;IAE1D,IAAI,IAAI,GAAG,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACjC,IAAI,SAAS,EAAE,CAAC;QACd,IAAI,GAAG,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACrC,CAAC;IACD,IAAI,GAAG,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAClC,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE;QACtB,kBAAkB,EAAE,QAAQ,CAAC,kBAAkB;QAC/C,eAAe,EAAE,QAAQ,CAAC,eAAe;KAC1C,CAAC,CAAC;IAEH,IAAI,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;QAC/C,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,OAAO,GAAG,gBAAgB,CAC9B,MAAM,EACN,OAAO,CAAC,SAAS,EACjB,QAAQ,CAAC,IAAI,EACb,IAAI,EACJ,QAAQ,CAAC,YAAY,CACtB,CAAC;QACF,IAAI,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAED,kCAAkC;IAClC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAClE,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,IAAI,IAAI,IAAI,CAAC;IAE1D,OAAO;QACL,IAAI,EAAE,IAAI;QACV,KAAK;QACL,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC;QACtB,IAAI;KACL,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,IAAI,CAAC,IAAY,EAAE,UAAuB,EAAE;IAC1D,OAAO,SAAS,CAAC,IAAI,EAAE;QACrB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAY,EACZ,UAAuB,EAAE;IAEzB,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACvC,IAAI,CAAC,OAAO,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAElC,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,IAAI,EAAE;QAC5C,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,KAAK,EAAE,OAAO,CAAC,KAAK;KACrB,CAAC,CAAC;IACH,uEAAuE;IACvE,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;IACxC,MAAM,UAAU,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IAC7C,IAAI,UAAU,IAAI,WAAW,EAAE,CAAC;QAC9B,OAAO,EAAE,GAAG,KAAK,EAAE,IAAI,EAAE,EAAE,GAAG,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,CAAC;IAC/D,CAAC;IACD,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,CAAC;IACrE,OAAO;QACL,IAAI,EAAE,OAAO;QACb,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,MAAM,EAAE,QAAQ,CAAC,OAAO,CAAC;QACzB,IAAI,EAAE,EAAE,GAAG,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE;KACxC,CAAC;AACJ,CAAC"}
|
package/dist/heuristics.d.ts
CHANGED
|
@@ -8,6 +8,20 @@ export declare function collapseBlankBlocks(blocks: Block[], cuts: CutCounts): B
|
|
|
8
8
|
* occurrences so the pattern is still visible.
|
|
9
9
|
*/
|
|
10
10
|
export declare function stripBoilerplate(blocks: Block[], cuts: CutCounts): Block[];
|
|
11
|
+
/**
|
|
12
|
+
* Drop quoted email reply chains: lines starting with ">", "On … wrote:",
|
|
13
|
+
* and classic "-----Original Message-----" / forwarded-message tails.
|
|
14
|
+
*/
|
|
15
|
+
export declare function stripQuotedReplies(blocks: Block[], cuts: CutCounts): Block[];
|
|
16
|
+
/**
|
|
17
|
+
* Drop Slack/Discord-ish chrome: reactions, join/leave system lines,
|
|
18
|
+
* lone timestamps, and "— unread —" dividers.
|
|
19
|
+
*/
|
|
20
|
+
export declare function stripChatChrome(blocks: Block[], cuts: CutCounts): Block[];
|
|
11
21
|
export declare function dedupeParagraphs(blocks: Block[], cuts: CutCounts): Block[];
|
|
12
|
-
export
|
|
22
|
+
export interface TidyOptions {
|
|
23
|
+
stripQuotedReplies?: boolean;
|
|
24
|
+
stripChatChrome?: boolean;
|
|
25
|
+
}
|
|
26
|
+
export declare function tidy(text: string, cuts: CutCounts, options?: TidyOptions): string;
|
|
13
27
|
//# sourceMappingURL=heuristics.d.ts.map
|
package/dist/heuristics.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"heuristics.d.ts","sourceRoot":"","sources":["../src/heuristics.ts"],"names":[],"mappings":"AAAA,OAAO,EAA2B,KAAK,KAAK,EAAE,MAAM,aAAa,CAAC;AAClE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"heuristics.d.ts","sourceRoot":"","sources":["../src/heuristics.ts"],"names":[],"mappings":"AAAA,OAAO,EAA2B,KAAK,KAAK,EAAE,MAAM,aAAa,CAAC;AAClE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAwC5C,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,GAAG,MAAM,CAQ/D;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,GAAG,MAAM,CAepE;AAeD,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,GAAG,KAAK,EAAE,CAsB7E;AAgCD;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,GAAG,KAAK,EAAE,CAiD1E;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,GAAG,KAAK,EAAE,CAyC5E;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,GAAG,KAAK,EAAE,CAoBzE;AAyBD,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,GAAG,KAAK,EAAE,CAyC1E;AAED,MAAM,WAAW,WAAW;IAC1B,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED,wBAAgB,IAAI,CAClB,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,SAAS,EACf,OAAO,GAAE,WAAgB,GACxB,MAAM,CAgBR"}
|
package/dist/heuristics.js
CHANGED
|
@@ -16,6 +16,18 @@ const CHROME_RE = [
|
|
|
16
16
|
/^\s*npm (info|http|timing|notice)\s/i,
|
|
17
17
|
/^\s*downloading (packages|metadata)\.{2,}\s*$/i,
|
|
18
18
|
];
|
|
19
|
+
const QUOTE_START_RE = [
|
|
20
|
+
/^\s*on .+ wrote:\s*$/i,
|
|
21
|
+
/^\s*-{2,}\s*original message\s*-{2,}\s*$/i,
|
|
22
|
+
/^\s*begin forwarded message\s*$/i,
|
|
23
|
+
];
|
|
24
|
+
const CHAT_CHROME_RE = [
|
|
25
|
+
/^\s*(reacted to|added a reaction|removed a reaction)\b/i,
|
|
26
|
+
/^\s*[👍👎❤️😂😮😢🔥🎉👏🤣✅❌]\s*(x?\d+)?\s*$/u,
|
|
27
|
+
/^\s*\[[^\]]*(joined|left|was added|was removed|changed the|renamed|pinned|unpinned)[^\]]*\]\s*$/i,
|
|
28
|
+
/^\s*(today|yesterday|\d{1,2}:\d{2}(\s?[ap]m)?)\s*$/i,
|
|
29
|
+
/^\s*—+\s*(new messages?|unread)\s*—+\s*$/i,
|
|
30
|
+
];
|
|
19
31
|
const DATA_URI_RE = /data:[a-zA-Z0-9.+-]+\/[a-zA-Z0-9.+-]+;base64,[A-Za-z0-9+/]+=*/g;
|
|
20
32
|
const HEAVY_BLOB_RE = /[A-Za-z0-9+/]{120,}={0,2}/g;
|
|
21
33
|
export function stripAnsi(text, cuts) {
|
|
@@ -161,6 +173,80 @@ export function stripBoilerplate(blocks, cuts) {
|
|
|
161
173
|
};
|
|
162
174
|
return blocks.map(mapBlock).filter((b) => b !== null);
|
|
163
175
|
}
|
|
176
|
+
/**
|
|
177
|
+
* Drop quoted email reply chains: lines starting with ">", "On … wrote:",
|
|
178
|
+
* and classic "-----Original Message-----" / forwarded-message tails.
|
|
179
|
+
*/
|
|
180
|
+
export function stripQuotedReplies(blocks, cuts) {
|
|
181
|
+
const out = [];
|
|
182
|
+
let dropRest = false;
|
|
183
|
+
for (const block of blocks) {
|
|
184
|
+
if (dropRest) {
|
|
185
|
+
if (block.type !== "blank" && block.type !== "code") {
|
|
186
|
+
const n = block.raw.split("\n").filter((l) => l.trim() !== "").length;
|
|
187
|
+
cuts.quotedReplies += Math.max(1, n);
|
|
188
|
+
}
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
if (block.type === "code" || block.type === "blank") {
|
|
192
|
+
out.push(block);
|
|
193
|
+
continue;
|
|
194
|
+
}
|
|
195
|
+
const kept = [];
|
|
196
|
+
for (const line of block.raw.split("\n")) {
|
|
197
|
+
const trimmed = line.trim();
|
|
198
|
+
if (QUOTE_START_RE.some((re) => re.test(trimmed))) {
|
|
199
|
+
cuts.quotedReplies += 1;
|
|
200
|
+
// Original Message / forwarded tails usually mean the rest is noise.
|
|
201
|
+
if (/original message|forwarded message/i.test(trimmed)) {
|
|
202
|
+
dropRest = true;
|
|
203
|
+
break;
|
|
204
|
+
}
|
|
205
|
+
continue;
|
|
206
|
+
}
|
|
207
|
+
if (/^>/.test(line)) {
|
|
208
|
+
cuts.quotedReplies += 1;
|
|
209
|
+
continue;
|
|
210
|
+
}
|
|
211
|
+
kept.push(line);
|
|
212
|
+
}
|
|
213
|
+
if (dropRest)
|
|
214
|
+
continue;
|
|
215
|
+
if (kept.length === 0)
|
|
216
|
+
continue;
|
|
217
|
+
if (kept.every((l) => l.trim() === ""))
|
|
218
|
+
continue;
|
|
219
|
+
out.push({ ...block, raw: kept.join("\n") });
|
|
220
|
+
}
|
|
221
|
+
return out;
|
|
222
|
+
}
|
|
223
|
+
/**
|
|
224
|
+
* Drop Slack/Discord-ish chrome: reactions, join/leave system lines,
|
|
225
|
+
* lone timestamps, and "— unread —" dividers.
|
|
226
|
+
*/
|
|
227
|
+
export function stripChatChrome(blocks, cuts) {
|
|
228
|
+
const out = [];
|
|
229
|
+
for (const block of blocks) {
|
|
230
|
+
if (block.type === "code" || block.type === "blank") {
|
|
231
|
+
out.push(block);
|
|
232
|
+
continue;
|
|
233
|
+
}
|
|
234
|
+
const kept = [];
|
|
235
|
+
for (const line of block.raw.split("\n")) {
|
|
236
|
+
if (CHAT_CHROME_RE.some((re) => re.test(line))) {
|
|
237
|
+
cuts.chatChrome += 1;
|
|
238
|
+
continue;
|
|
239
|
+
}
|
|
240
|
+
kept.push(line);
|
|
241
|
+
}
|
|
242
|
+
if (kept.length === 0)
|
|
243
|
+
continue;
|
|
244
|
+
if (kept.every((l) => l.trim() === ""))
|
|
245
|
+
continue;
|
|
246
|
+
out.push({ ...block, raw: kept.join("\n") });
|
|
247
|
+
}
|
|
248
|
+
return out;
|
|
249
|
+
}
|
|
164
250
|
function normalizeParagraph(text) {
|
|
165
251
|
return text
|
|
166
252
|
.toLowerCase()
|
|
@@ -226,9 +312,17 @@ export function dedupeParagraphs(blocks, cuts) {
|
|
|
226
312
|
}
|
|
227
313
|
return out;
|
|
228
314
|
}
|
|
229
|
-
export function tidy(text, cuts) {
|
|
315
|
+
export function tidy(text, cuts, options = {}) {
|
|
230
316
|
let blocks = parseBlocks(text);
|
|
231
317
|
blocks = collapseBlankBlocks(blocks, cuts);
|
|
318
|
+
if (options.stripQuotedReplies) {
|
|
319
|
+
blocks = stripQuotedReplies(blocks, cuts);
|
|
320
|
+
blocks = collapseBlankBlocks(blocks, cuts);
|
|
321
|
+
}
|
|
322
|
+
if (options.stripChatChrome) {
|
|
323
|
+
blocks = stripChatChrome(blocks, cuts);
|
|
324
|
+
blocks = collapseBlankBlocks(blocks, cuts);
|
|
325
|
+
}
|
|
232
326
|
blocks = stripBoilerplate(blocks, cuts);
|
|
233
327
|
blocks = collapseBlankBlocks(blocks, cuts);
|
|
234
328
|
blocks = dedupeParagraphs(blocks, cuts);
|
package/dist/heuristics.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"heuristics.js","sourceRoot":"","sources":["../src/heuristics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAc,MAAM,aAAa,CAAC;AAGlE,MAAM,OAAO,GAAG,gFAAgF,CAAC;AAEjG,MAAM,YAAY,GAAG;IACnB,gDAAgD;IAChD,wCAAwC;IACxC,0EAA0E;IAC1E,+BAA+B;IAC/B,iDAAiD;IACjD,8CAA8C;IAC9C,yBAAyB;IACzB,kCAAkC;IAClC,yCAAyC;CAC1C,CAAC;AAEF,MAAM,SAAS,GAAG;IAChB,8CAA8C;IAC9C,sCAAsC;IACtC,gDAAgD;CACjD,CAAC;AAEF,MAAM,WAAW,GACf,gEAAgE,CAAC;AACnE,MAAM,aAAa,GAAG,4BAA4B,CAAC;AAEnD,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,IAAe;IACrD,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,EAAE;QACtC,CAAC,EAAE,CAAC;QACJ,OAAO,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC;IACxB,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,IAAe;IAC1D,MAAM,OAAO,GAAG,CAAC,IAAY,EAAU,EAAE;QACvC,IAAI,CAAC,iBAAiB,IAAI,CAAC,CAAC;QAC5B,OAAO,kBAAkB,IAAI,CAAC,MAAM,SAAS,CAAC;IAChD,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;IAChF,OAAO,IAAI,CAAC,OAAO,CACjB,oCAAoC,EACpC,CAAC,GAAG,EAAE,EAAE;QACN,MAAM,KAAK,GAAG,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnE,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/C,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,IAAI,CAAC,iBAAiB,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;QACjE,OAAO,kBAAkB,KAAK,SAAS,CAAC;IAC1C,CAAC,CACF,CAAC;AACJ,CAAC;AAED,SAAS,uBAAuB,CAAC,GAAW,EAAE,IAAe;IAC3D,MAAM,IAAI,GAAG,GAAG;SACb,OAAO,CAAC,WAAW,EAAE,CAAC,EAAE,EAAE,EAAE;QAC3B,IAAI,EAAE,CAAC,MAAM;YAAE,IAAI,CAAC,cAAc,IAAI,CAAC,CAAC;QACxC,OAAO,EAAE,CAAC;IACZ,CAAC,CAAC;SACD,OAAO,CAAC,YAAY,EAAE,GAAG,EAAE;QAC1B,IAAI,CAAC,cAAc,IAAI,CAAC,CAAC;QACzB,OAAO,GAAG,CAAC;IACb,CAAC,CAAC,CAAC;IACL,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,MAAe,EAAE,IAAe;IAClE,MAAM,GAAG,GAAY,EAAE,CAAC;IACxB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;YAC1C,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAC/B,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,IAAI,KAAK,OAAO,EAAE,CAAC;gBAC1C,IAAI,CAAC,cAAc,IAAI,CAAC,CAAC;gBACzB,SAAS;YACX,CAAC;YACD,IAAI,IAAI,GAAG,CAAC;gBAAE,IAAI,CAAC,cAAc,IAAI,IAAI,GAAG,CAAC,CAAC;YAC9C,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;YACvC,SAAS;QACX,CAAC;QACD,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC1B,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChB,SAAS;QACX,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,EAAE,GAAG,EAAE,uBAAuB,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;IACxE,CAAC;IACD,OAAO,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,IAAI,KAAK,OAAO;QAAE,GAAG,CAAC,GAAG,EAAE,CAAC;IACtE,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,eAAe,CAAC,IAAY;IACnC,OAAO,IAAI;SACR,WAAW,EAAE;SACb,OAAO,CAAC,6EAA6E,EAAE,MAAM,CAAC;SAC9F,OAAO,CACN,gEAAgE,EAChE,QAAQ,CACT;SACA,OAAO,CAAC,eAAe,EAAE,OAAO,CAAC;SACjC,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC;SAC1B,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC;SAC5B,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,OAAO,YAAY,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAY;IACpC,OAAO,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;AAC9C,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAY;IACpC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IACtB,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACjC,IAAI,gBAAgB,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACxC,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AAC/C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAe,EAAE,IAAe;IAC/D,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO;YAAE,SAAS;QAC9D,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACzC,MAAM,EAAE,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;YACjC,IAAI,EAAE;gBAAE,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;IACvC,KAAK,MAAM,EAAE,IAAI,YAAY;QAAE,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAErE,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;IACvC,MAAM,YAAY,GAAG,CAAC,CAAC;IAEvB,MAAM,QAAQ,GAAG,CAAC,KAAY,EAAgB,EAAE;QAC9C,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO;YAAE,OAAO,KAAK,CAAC;QAClE,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC1B,IAAI,CAAC,gBAAgB,IAAI,CAAC,CAAC;YAC3B,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACzC,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC;gBAChD,IAAI,CAAC,gBAAgB,IAAI,CAAC,CAAC;gBAC3B,SAAS;YACX,CAAC;YACD,MAAM,EAAE,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;YACjC,MAAM,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACzC,MAAM,WAAW,GAAG,OAAO,CAAC,EAAE,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;YAC9C,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;YACtC,IAAI,WAAW,IAAI,CAAC,MAAM,IAAI,KAAK,IAAI,CAAC,CAAC,EAAE,CAAC;gBAC1C,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;gBACtC,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC;gBACtC,IAAI,WAAW,IAAI,GAAG,EAAE,CAAC;oBACvB,IAAI,CAAC,gBAAgB,IAAI,CAAC,CAAC;oBAC3B,SAAS;gBACX,CAAC;gBACD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC;YAChC,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClB,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QACnC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC;YAAE,OAAO,IAAI,CAAC;QACpD,OAAO,EAAE,GAAG,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;IAC5C,CAAC,CAAC;IAEF,OAAO,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAc,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;AACpE,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,OAAO,IAAI;SACR,WAAW,EAAE;SACb,OAAO,CAAC,iBAAiB,EAAE,OAAO,CAAC;SACnC,OAAO,CAAC,2BAA2B,EAAE,GAAG,CAAC;SACzC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,OAAO,CAAC,CAAS,EAAE,CAAS;IACnC,MAAM,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACxC,MAAM,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACxC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACjD,MAAM,EAAE,GAAG,IAAI,GAAG,CAAC,EAAE,CAAC,CAAC;IACvB,MAAM,EAAE,GAAG,IAAI,GAAG,CAAC,EAAE,CAAC,CAAC;IACvB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,CAAC,IAAI,EAAE;QAAE,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,KAAK,EAAE,CAAC;IAC3C,MAAM,KAAK,GAAG,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC,IAAI,GAAG,KAAK,CAAC;IACxC,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC;AACzC,CAAC;AAED,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAE9B,MAAM,UAAU,gBAAgB,CAAC,MAAe,EAAE,IAAe;IAC/D,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IACpC,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,GAAG,GAAY,EAAE,CAAC;IAExB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAChF,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChB,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,kBAAkB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC3C,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChB,SAAS;QACX,CAAC;QACD,IAAI,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,IAAI,CAAC,mBAAmB,IAAI,CAAC,CAAC;YAC9B,SAAS;QACX,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;QACrC,IAAI,GAAG,GAAG,KAAK,CAAC;QAChB,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YACf,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;gBAC7B,IAAI,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,gBAAgB,EAAE,CAAC;oBAC5C,GAAG,GAAG,IAAI,CAAC;oBACX,MAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;QACD,IAAI,GAAG,EAAE,CAAC;YACR,IAAI,CAAC,mBAAmB,IAAI,CAAC,CAAC;YAC9B,SAAS;QACX,CAAC;QACD,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpB,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YACf,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrB,IAAI,SAAS,CAAC,MAAM,GAAG,EAAE;gBAAE,SAAS,CAAC,KAAK,EAAE,CAAC;QAC/C,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,IAAI,CAAC,IAAY,EAAE,IAAe;IAChD,IAAI,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC3C,MAAM,GAAG,gBAAgB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACxC,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC3C,MAAM,GAAG,gBAAgB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACxC,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC3C,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC"}
|
|
1
|
+
{"version":3,"file":"heuristics.js","sourceRoot":"","sources":["../src/heuristics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAc,MAAM,aAAa,CAAC;AAGlE,MAAM,OAAO,GAAG,gFAAgF,CAAC;AAEjG,MAAM,YAAY,GAAG;IACnB,gDAAgD;IAChD,wCAAwC;IACxC,0EAA0E;IAC1E,+BAA+B;IAC/B,iDAAiD;IACjD,8CAA8C;IAC9C,yBAAyB;IACzB,kCAAkC;IAClC,yCAAyC;CAC1C,CAAC;AAEF,MAAM,SAAS,GAAG;IAChB,8CAA8C;IAC9C,sCAAsC;IACtC,gDAAgD;CACjD,CAAC;AAEF,MAAM,cAAc,GAAG;IACrB,uBAAuB;IACvB,2CAA2C;IAC3C,kCAAkC;CACnC,CAAC;AAEF,MAAM,cAAc,GAAG;IACrB,yDAAyD;IACzD,8CAA8C;IAC9C,kGAAkG;IAClG,qDAAqD;IACrD,2CAA2C;CAC5C,CAAC;AAEF,MAAM,WAAW,GACf,gEAAgE,CAAC;AACnE,MAAM,aAAa,GAAG,4BAA4B,CAAC;AAEnD,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,IAAe;IACrD,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,EAAE;QACtC,CAAC,EAAE,CAAC;QACJ,OAAO,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC;IACxB,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,IAAe;IAC1D,MAAM,OAAO,GAAG,CAAC,IAAY,EAAU,EAAE;QACvC,IAAI,CAAC,iBAAiB,IAAI,CAAC,CAAC;QAC5B,OAAO,kBAAkB,IAAI,CAAC,MAAM,SAAS,CAAC;IAChD,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,OAAO,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;IAChF,OAAO,IAAI,CAAC,OAAO,CACjB,oCAAoC,EACpC,CAAC,GAAG,EAAE,EAAE;QACN,MAAM,KAAK,GAAG,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnE,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/C,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,IAAI,CAAC,iBAAiB,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;QACjE,OAAO,kBAAkB,KAAK,SAAS,CAAC;IAC1C,CAAC,CACF,CAAC;AACJ,CAAC;AAED,SAAS,uBAAuB,CAAC,GAAW,EAAE,IAAe;IAC3D,MAAM,IAAI,GAAG,GAAG;SACb,OAAO,CAAC,WAAW,EAAE,CAAC,EAAE,EAAE,EAAE;QAC3B,IAAI,EAAE,CAAC,MAAM;YAAE,IAAI,CAAC,cAAc,IAAI,CAAC,CAAC;QACxC,OAAO,EAAE,CAAC;IACZ,CAAC,CAAC;SACD,OAAO,CAAC,YAAY,EAAE,GAAG,EAAE;QAC1B,IAAI,CAAC,cAAc,IAAI,CAAC,CAAC;QACzB,OAAO,GAAG,CAAC;IACb,CAAC,CAAC,CAAC;IACL,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,MAAe,EAAE,IAAe;IAClE,MAAM,GAAG,GAAY,EAAE,CAAC;IACxB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;YAC1C,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAC/B,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,IAAI,KAAK,OAAO,EAAE,CAAC;gBAC1C,IAAI,CAAC,cAAc,IAAI,CAAC,CAAC;gBACzB,SAAS;YACX,CAAC;YACD,IAAI,IAAI,GAAG,CAAC;gBAAE,IAAI,CAAC,cAAc,IAAI,IAAI,GAAG,CAAC,CAAC;YAC9C,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;YACvC,SAAS;QACX,CAAC;QACD,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC1B,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChB,SAAS;QACX,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,EAAE,GAAG,EAAE,uBAAuB,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;IACxE,CAAC;IACD,OAAO,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,IAAI,KAAK,OAAO;QAAE,GAAG,CAAC,GAAG,EAAE,CAAC;IACtE,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,eAAe,CAAC,IAAY;IACnC,OAAO,IAAI;SACR,WAAW,EAAE;SACb,OAAO,CAAC,6EAA6E,EAAE,MAAM,CAAC;SAC9F,OAAO,CACN,gEAAgE,EAChE,QAAQ,CACT;SACA,OAAO,CAAC,eAAe,EAAE,OAAO,CAAC;SACjC,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC;SAC1B,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC;SAC5B,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,OAAO,YAAY,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAY;IACpC,OAAO,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;AAC9C,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAY;IACpC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IACtB,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACjC,IAAI,gBAAgB,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACxC,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AAC/C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAe,EAAE,IAAe;IAC/D,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO;YAAE,SAAS;QAC9D,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACzC,MAAM,EAAE,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;YACjC,IAAI,EAAE;gBAAE,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;IACvC,KAAK,MAAM,EAAE,IAAI,YAAY;QAAE,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAErE,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;IACvC,MAAM,YAAY,GAAG,CAAC,CAAC;IAEvB,MAAM,QAAQ,GAAG,CAAC,KAAY,EAAgB,EAAE;QAC9C,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO;YAAE,OAAO,KAAK,CAAC;QAClE,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC1B,IAAI,CAAC,gBAAgB,IAAI,CAAC,CAAC;YAC3B,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACzC,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC;gBAChD,IAAI,CAAC,gBAAgB,IAAI,CAAC,CAAC;gBAC3B,SAAS;YACX,CAAC;YACD,MAAM,EAAE,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;YACjC,MAAM,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACzC,MAAM,WAAW,GAAG,OAAO,CAAC,EAAE,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;YAC9C,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;YACtC,IAAI,WAAW,IAAI,CAAC,MAAM,IAAI,KAAK,IAAI,CAAC,CAAC,EAAE,CAAC;gBAC1C,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;gBACtC,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC;gBACtC,IAAI,WAAW,IAAI,GAAG,EAAE,CAAC;oBACvB,IAAI,CAAC,gBAAgB,IAAI,CAAC,CAAC;oBAC3B,SAAS;gBACX,CAAC;gBACD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC;YAChC,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClB,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QACnC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC;YAAE,OAAO,IAAI,CAAC;QACpD,OAAO,EAAE,GAAG,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;IAC5C,CAAC,CAAC;IAEF,OAAO,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAc,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;AACpE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,MAAe,EAAE,IAAe;IACjE,MAAM,GAAG,GAAY,EAAE,CAAC;IACxB,IAAI,QAAQ,GAAG,KAAK,CAAC;IAErB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACpD,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC;gBACtE,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACvC,CAAC;YACD,SAAS;QACX,CAAC;QACD,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YACpD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChB,SAAS;QACX,CAAC;QAED,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACzC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;gBAClD,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC;gBACxB,qEAAqE;gBACrE,IAAI,qCAAqC,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;oBACxD,QAAQ,GAAG,IAAI,CAAC;oBAChB,MAAM;gBACR,CAAC;gBACD,SAAS;YACX,CAAC;YACD,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACpB,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC;gBACxB,SAAS;YACX,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClB,CAAC;QACD,IAAI,QAAQ;YAAE,SAAS;QACvB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAChC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC;YAAE,SAAS;QACjD,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,MAAe,EAAE,IAAe;IAC9D,MAAM,GAAG,GAAY,EAAE,CAAC;IACxB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YACpD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChB,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACzC,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;gBAC/C,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;gBACrB,SAAS;YACX,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClB,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAChC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC;YAAE,SAAS;QACjD,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,OAAO,IAAI;SACR,WAAW,EAAE;SACb,OAAO,CAAC,iBAAiB,EAAE,OAAO,CAAC;SACnC,OAAO,CAAC,2BAA2B,EAAE,GAAG,CAAC;SACzC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,OAAO,CAAC,CAAS,EAAE,CAAS;IACnC,MAAM,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACxC,MAAM,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACxC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACjD,MAAM,EAAE,GAAG,IAAI,GAAG,CAAC,EAAE,CAAC,CAAC;IACvB,MAAM,EAAE,GAAG,IAAI,GAAG,CAAC,EAAE,CAAC,CAAC;IACvB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,CAAC,IAAI,EAAE;QAAE,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,KAAK,EAAE,CAAC;IAC3C,MAAM,KAAK,GAAG,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC,IAAI,GAAG,KAAK,CAAC;IACxC,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC;AACzC,CAAC;AAED,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAE9B,MAAM,UAAU,gBAAgB,CAAC,MAAe,EAAE,IAAe;IAC/D,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IACpC,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,GAAG,GAAY,EAAE,CAAC;IAExB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAChF,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChB,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,kBAAkB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC3C,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAChB,SAAS;QACX,CAAC;QACD,IAAI,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,IAAI,CAAC,mBAAmB,IAAI,CAAC,CAAC;YAC9B,SAAS;QACX,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;QACrC,IAAI,GAAG,GAAG,KAAK,CAAC;QAChB,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YACf,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;gBAC7B,IAAI,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,gBAAgB,EAAE,CAAC;oBAC5C,GAAG,GAAG,IAAI,CAAC;oBACX,MAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;QACD,IAAI,GAAG,EAAE,CAAC;YACR,IAAI,CAAC,mBAAmB,IAAI,CAAC,CAAC;YAC9B,SAAS;QACX,CAAC;QACD,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpB,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YACf,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrB,IAAI,SAAS,CAAC,MAAM,GAAG,EAAE;gBAAE,SAAS,CAAC,KAAK,EAAE,CAAC;QAC/C,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAOD,MAAM,UAAU,IAAI,CAClB,IAAY,EACZ,IAAe,EACf,UAAuB,EAAE;IAEzB,IAAI,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC3C,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAC/B,MAAM,GAAG,kBAAkB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAC1C,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC7C,CAAC;IACD,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;QAC5B,MAAM,GAAG,eAAe,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACvC,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC7C,CAAC;IACD,MAAM,GAAG,gBAAgB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACxC,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC3C,MAAM,GAAG,gBAAgB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACxC,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC3C,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC"}
|
package/dist/html.d.ts
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { CutCounts } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Strip HTML chrome (scripts, styles, comments, tags) into readable prose.
|
|
4
|
+
* Counts each removed tag/script/style/comment block toward `htmlChrome`.
|
|
5
|
+
*/
|
|
6
|
+
export declare function stripHtmlChrome(text: string, cuts: CutCounts): string;
|
|
7
|
+
//# sourceMappingURL=html.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"html.d.ts","sourceRoot":"","sources":["../src/html.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAgC5C;;;GAGG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,GAAG,MAAM,CAmCrE"}
|
package/dist/html.js
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
const SCRIPT_RE = /<script\b[^>]*>[\s\S]*?<\/script\s*>/gi;
|
|
2
|
+
const STYLE_RE = /<style\b[^>]*>[\s\S]*?<\/style\s*>/gi;
|
|
3
|
+
const NOSCRIPT_RE = /<noscript\b[^>]*>[\s\S]*?<\/noscript\s*>/gi;
|
|
4
|
+
const COMMENT_RE = /<!--[\s\S]*?-->/g;
|
|
5
|
+
const BLOCK_TAG_RE = /<\/?(?:html|head|body|div|p|br|hr|li|ul|ol|tr|td|th|table|section|article|header|footer|nav|main|h[1-6]|blockquote|pre|form|aside)\b[^>]*>/gi;
|
|
6
|
+
const ANY_TAG_RE = /<\/?[a-zA-Z][^>]*>/g;
|
|
7
|
+
const ENTITIES = {
|
|
8
|
+
amp: "&",
|
|
9
|
+
lt: "<",
|
|
10
|
+
gt: ">",
|
|
11
|
+
quot: '"',
|
|
12
|
+
apos: "'",
|
|
13
|
+
nbsp: " ",
|
|
14
|
+
};
|
|
15
|
+
function decodeEntities(text) {
|
|
16
|
+
return text
|
|
17
|
+
.replace(/&#x([0-9a-fA-F]+);/g, (_, hex) => {
|
|
18
|
+
const code = Number.parseInt(hex, 16);
|
|
19
|
+
return Number.isFinite(code) ? String.fromCodePoint(code) : _;
|
|
20
|
+
})
|
|
21
|
+
.replace(/&#(\d+);/g, (_, dec) => {
|
|
22
|
+
const code = Number(dec);
|
|
23
|
+
return Number.isFinite(code) ? String.fromCodePoint(code) : _;
|
|
24
|
+
})
|
|
25
|
+
.replace(/&([a-zA-Z]+);/g, (full, name) => ENTITIES[name.toLowerCase()] ?? full);
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Strip HTML chrome (scripts, styles, comments, tags) into readable prose.
|
|
29
|
+
* Counts each removed tag/script/style/comment block toward `htmlChrome`.
|
|
30
|
+
*/
|
|
31
|
+
export function stripHtmlChrome(text, cuts) {
|
|
32
|
+
if (!/<[a-zA-Z!/]/.test(text))
|
|
33
|
+
return text;
|
|
34
|
+
let n = 0;
|
|
35
|
+
const bump = () => {
|
|
36
|
+
n += 1;
|
|
37
|
+
return "";
|
|
38
|
+
};
|
|
39
|
+
let next = text
|
|
40
|
+
.replace(SCRIPT_RE, bump)
|
|
41
|
+
.replace(STYLE_RE, bump)
|
|
42
|
+
.replace(NOSCRIPT_RE, bump)
|
|
43
|
+
.replace(COMMENT_RE, bump);
|
|
44
|
+
next = next.replace(BLOCK_TAG_RE, (tag) => {
|
|
45
|
+
n += 1;
|
|
46
|
+
const lower = tag.toLowerCase();
|
|
47
|
+
if (lower.startsWith("<br") || lower.startsWith("<hr") || lower.startsWith("</")) {
|
|
48
|
+
return "\n";
|
|
49
|
+
}
|
|
50
|
+
return "\n";
|
|
51
|
+
});
|
|
52
|
+
next = next.replace(ANY_TAG_RE, () => {
|
|
53
|
+
n += 1;
|
|
54
|
+
return "";
|
|
55
|
+
});
|
|
56
|
+
next = decodeEntities(next);
|
|
57
|
+
// Collapse spaces introduced by tag removal, but keep intentional newlines.
|
|
58
|
+
next = next.replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n");
|
|
59
|
+
cuts.htmlChrome += n;
|
|
60
|
+
return next;
|
|
61
|
+
}
|
|
62
|
+
//# sourceMappingURL=html.js.map
|
package/dist/html.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"html.js","sourceRoot":"","sources":["../src/html.ts"],"names":[],"mappings":"AAEA,MAAM,SAAS,GAAG,wCAAwC,CAAC;AAC3D,MAAM,QAAQ,GAAG,sCAAsC,CAAC;AACxD,MAAM,WAAW,GAAG,4CAA4C,CAAC;AACjE,MAAM,UAAU,GAAG,kBAAkB,CAAC;AACtC,MAAM,YAAY,GAChB,8IAA8I,CAAC;AACjJ,MAAM,UAAU,GAAG,qBAAqB,CAAC;AAEzC,MAAM,QAAQ,GAA2B;IACvC,GAAG,EAAE,GAAG;IACR,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,GAAG;IACP,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,GAAG;CACV,CAAC;AAEF,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,IAAI;SACR,OAAO,CAAC,qBAAqB,EAAE,CAAC,CAAC,EAAE,GAAW,EAAE,EAAE;QACjD,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACtC,OAAO,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,CAAC,CAAC;SACD,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,GAAW,EAAE,EAAE;QACvC,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QACzB,OAAO,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,CAAC,CAAC;SACD,OAAO,CAAC,gBAAgB,EAAE,CAAC,IAAI,EAAE,IAAY,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC;AAC7F,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,IAAY,EAAE,IAAe;IAC3D,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAE3C,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,MAAM,IAAI,GAAG,GAAW,EAAE;QACxB,CAAC,IAAI,CAAC,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC,CAAC;IAEF,IAAI,IAAI,GAAG,IAAI;SACZ,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC;SACxB,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;SACvB,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC;SAC1B,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IAE7B,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC,GAAG,EAAE,EAAE;QACxC,CAAC,IAAI,CAAC,CAAC;QACP,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QAChC,IAAI,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACjF,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,EAAE;QACnC,CAAC,IAAI,CAAC,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC;IAEH,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IAC5B,4EAA4E;IAC5E,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAElE,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;IACrB,OAAO,IAAI,CAAC;AACd,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
export { parseBlocks, joinBlocks, type Block, type BlockType } from "./blocks.js";
|
|
2
2
|
export { diet, dietAsync } from "./diet.js";
|
|
3
3
|
export { deepenText, DeepenError, hasAnyKey, resolveProvider } from "./llm.js";
|
|
4
|
+
export { stripHtmlChrome } from "./html.js";
|
|
5
|
+
export { PRESETS, PRESET_NAMES, isPresetName, resolvePreset, type PresetConfig, type PresetName, } from "./presets.js";
|
|
4
6
|
export { estimateTokens, snapshot } from "./tokens.js";
|
|
5
|
-
export { formatHumanReport, toJsonReport, type JsonReport } from "./report.js";
|
|
7
|
+
export { formatHumanReport, formatRemovedDiff, shouldUseColor, toJsonReport, type JsonReport, type ReportStyle, } from "./report.js";
|
|
6
8
|
export { KEEP_CATEGORIES, emptyCuts, type CutCounts, type DietOptions, type DietResult, type KeepCategory, type LlmProvider, type SizeSnapshot, } from "./types.js";
|
|
7
9
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,KAAK,KAAK,EAAE,KAAK,SAAS,EAAE,MAAM,aAAa,CAAC;AAClF,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAC/E,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,KAAK,KAAK,EAAE,KAAK,SAAS,EAAE,MAAM,aAAa,CAAC;AAClF,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAC/E,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,EACL,OAAO,EACP,YAAY,EACZ,YAAY,EACZ,aAAa,EACb,KAAK,YAAY,EACjB,KAAK,UAAU,GAChB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,EACL,iBAAiB,EACjB,iBAAiB,EACjB,cAAc,EACd,YAAY,EACZ,KAAK,UAAU,EACf,KAAK,WAAW,GACjB,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,eAAe,EACf,SAAS,EACT,KAAK,SAAS,EACd,KAAK,WAAW,EAChB,KAAK,UAAU,EACf,KAAK,YAAY,EACjB,KAAK,WAAW,EAChB,KAAK,YAAY,GAClB,MAAM,YAAY,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
export { parseBlocks, joinBlocks } from "./blocks.js";
|
|
2
2
|
export { diet, dietAsync } from "./diet.js";
|
|
3
3
|
export { deepenText, DeepenError, hasAnyKey, resolveProvider } from "./llm.js";
|
|
4
|
+
export { stripHtmlChrome } from "./html.js";
|
|
5
|
+
export { PRESETS, PRESET_NAMES, isPresetName, resolvePreset, } from "./presets.js";
|
|
4
6
|
export { estimateTokens, snapshot } from "./tokens.js";
|
|
5
|
-
export { formatHumanReport, toJsonReport } from "./report.js";
|
|
7
|
+
export { formatHumanReport, formatRemovedDiff, shouldUseColor, toJsonReport, } from "./report.js";
|
|
6
8
|
export { KEEP_CATEGORIES, emptyCuts, } from "./types.js";
|
|
7
9
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,UAAU,EAA8B,MAAM,aAAa,CAAC;AAClF,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAC/E,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,UAAU,EAA8B,MAAM,aAAa,CAAC;AAClF,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAC/E,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,EACL,OAAO,EACP,YAAY,EACZ,YAAY,EACZ,aAAa,GAGd,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,EACL,iBAAiB,EACjB,iBAAiB,EACjB,cAAc,EACd,YAAY,GAGb,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,eAAe,EACf,SAAS,GAOV,MAAM,YAAY,CAAC"}
|