@smeltjs/core 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +55 -0
- package/THIRD-PARTY.md +140 -0
- package/dist/apply.d.ts +145 -0
- package/dist/apply.d.ts.map +1 -0
- package/dist/apply.js +244 -0
- package/dist/apply.js.map +1 -0
- package/dist/cache/prefix.d.ts +164 -0
- package/dist/cache/prefix.d.ts.map +1 -0
- package/dist/cache/prefix.js +361 -0
- package/dist/cache/prefix.js.map +1 -0
- package/dist/cli/args.d.ts +71 -0
- package/dist/cli/args.d.ts.map +1 -0
- package/dist/cli/args.js +308 -0
- package/dist/cli/args.js.map +1 -0
- package/dist/cli/bin.d.ts +3 -0
- package/dist/cli/bin.d.ts.map +1 -0
- package/dist/cli/bin.js +161 -0
- package/dist/cli/bin.js.map +1 -0
- package/dist/cli/config.d.ts +73 -0
- package/dist/cli/config.d.ts.map +1 -0
- package/dist/cli/config.js +144 -0
- package/dist/cli/config.js.map +1 -0
- package/dist/cli/init.d.ts +56 -0
- package/dist/cli/init.d.ts.map +1 -0
- package/dist/cli/init.js +449 -0
- package/dist/cli/init.js.map +1 -0
- package/dist/cli/report.d.ts +44 -0
- package/dist/cli/report.d.ts.map +1 -0
- package/dist/cli/report.js +113 -0
- package/dist/cli/report.js.map +1 -0
- package/dist/cli/resolve.d.ts +103 -0
- package/dist/cli/resolve.d.ts.map +1 -0
- package/dist/cli/resolve.js +106 -0
- package/dist/cli/resolve.js.map +1 -0
- package/dist/cli/run.d.ts +99 -0
- package/dist/cli/run.d.ts.map +1 -0
- package/dist/cli/run.js +351 -0
- package/dist/cli/run.js.map +1 -0
- package/dist/detect.d.ts +16 -0
- package/dist/detect.d.ts.map +1 -0
- package/dist/detect.js +18 -0
- package/dist/detect.js.map +1 -0
- package/dist/errors.d.ts +88 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +107 -0
- package/dist/errors.js.map +1 -0
- package/dist/hash.d.ts +13 -0
- package/dist/hash.d.ts.map +1 -0
- package/dist/hash.js +16 -0
- package/dist/hash.js.map +1 -0
- package/dist/index.d.ts +121 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +93 -0
- package/dist/index.js.map +1 -0
- package/dist/lang/bash.d.ts +3 -0
- package/dist/lang/bash.d.ts.map +1 -0
- package/dist/lang/bash.js +28 -0
- package/dist/lang/bash.js.map +1 -0
- package/dist/lang/c.d.ts +3 -0
- package/dist/lang/c.d.ts.map +1 -0
- package/dist/lang/c.js +39 -0
- package/dist/lang/c.js.map +1 -0
- package/dist/lang/c_sharp.d.ts +3 -0
- package/dist/lang/c_sharp.d.ts.map +1 -0
- package/dist/lang/c_sharp.js +37 -0
- package/dist/lang/c_sharp.js.map +1 -0
- package/dist/lang/cpp.d.ts +3 -0
- package/dist/lang/cpp.d.ts.map +1 -0
- package/dist/lang/cpp.js +46 -0
- package/dist/lang/cpp.js.map +1 -0
- package/dist/lang/go.d.ts +3 -0
- package/dist/lang/go.d.ts.map +1 -0
- package/dist/lang/go.js +31 -0
- package/dist/lang/go.js.map +1 -0
- package/dist/lang/java.d.ts +3 -0
- package/dist/lang/java.d.ts.map +1 -0
- package/dist/lang/java.js +34 -0
- package/dist/lang/java.js.map +1 -0
- package/dist/lang/javascript.d.ts +3 -0
- package/dist/lang/javascript.d.ts.map +1 -0
- package/dist/lang/javascript.js +28 -0
- package/dist/lang/javascript.js.map +1 -0
- package/dist/lang/kotlin.d.ts +3 -0
- package/dist/lang/kotlin.d.ts.map +1 -0
- package/dist/lang/kotlin.js +37 -0
- package/dist/lang/kotlin.js.map +1 -0
- package/dist/lang/php.d.ts +3 -0
- package/dist/lang/php.d.ts.map +1 -0
- package/dist/lang/php.js +37 -0
- package/dist/lang/php.js.map +1 -0
- package/dist/lang/profile.d.ts +130 -0
- package/dist/lang/profile.d.ts.map +1 -0
- package/dist/lang/profile.js +2 -0
- package/dist/lang/profile.js.map +1 -0
- package/dist/lang/python.d.ts +3 -0
- package/dist/lang/python.d.ts.map +1 -0
- package/dist/lang/python.js +33 -0
- package/dist/lang/python.js.map +1 -0
- package/dist/lang/registry.d.ts +31 -0
- package/dist/lang/registry.d.ts.map +1 -0
- package/dist/lang/registry.js +89 -0
- package/dist/lang/registry.js.map +1 -0
- package/dist/lang/ruby.d.ts +3 -0
- package/dist/lang/ruby.d.ts.map +1 -0
- package/dist/lang/ruby.js +47 -0
- package/dist/lang/ruby.js.map +1 -0
- package/dist/lang/rust.d.ts +3 -0
- package/dist/lang/rust.d.ts.map +1 -0
- package/dist/lang/rust.js +45 -0
- package/dist/lang/rust.js.map +1 -0
- package/dist/lang/swift.d.ts +3 -0
- package/dist/lang/swift.d.ts.map +1 -0
- package/dist/lang/swift.js +34 -0
- package/dist/lang/swift.js.map +1 -0
- package/dist/lang/tsx.d.ts +4 -0
- package/dist/lang/tsx.d.ts.map +1 -0
- package/dist/lang/tsx.js +11 -0
- package/dist/lang/tsx.js.map +1 -0
- package/dist/lang/typescript.d.ts +18 -0
- package/dist/lang/typescript.d.ts.map +1 -0
- package/dist/lang/typescript.js +61 -0
- package/dist/lang/typescript.js.map +1 -0
- package/dist/net/policy.d.ts +49 -0
- package/dist/net/policy.d.ts.map +1 -0
- package/dist/net/policy.js +120 -0
- package/dist/net/policy.js.map +1 -0
- package/dist/plan/grammar.d.ts +38 -0
- package/dist/plan/grammar.d.ts.map +1 -0
- package/dist/plan/grammar.js +85 -0
- package/dist/plan/grammar.js.map +1 -0
- package/dist/plan/lexical.d.ts +51 -0
- package/dist/plan/lexical.d.ts.map +1 -0
- package/dist/plan/lexical.js +167 -0
- package/dist/plan/lexical.js.map +1 -0
- package/dist/plan/planners.d.ts +35 -0
- package/dist/plan/planners.d.ts.map +1 -0
- package/dist/plan/planners.js +26 -0
- package/dist/plan/planners.js.map +1 -0
- package/dist/plan/structural.d.ts +68 -0
- package/dist/plan/structural.d.ts.map +1 -0
- package/dist/plan/structural.js +458 -0
- package/dist/plan/structural.js.map +1 -0
- package/dist/repomap/cache.d.ts +53 -0
- package/dist/repomap/cache.d.ts.map +1 -0
- package/dist/repomap/cache.js +139 -0
- package/dist/repomap/cache.js.map +1 -0
- package/dist/repomap/map.d.ts +161 -0
- package/dist/repomap/map.d.ts.map +1 -0
- package/dist/repomap/map.js +302 -0
- package/dist/repomap/map.js.map +1 -0
- package/dist/repomap/rank.d.ts +52 -0
- package/dist/repomap/rank.d.ts.map +1 -0
- package/dist/repomap/rank.js +155 -0
- package/dist/repomap/rank.js.map +1 -0
- package/dist/repomap/tags.d.ts +47 -0
- package/dist/repomap/tags.d.ts.map +1 -0
- package/dist/repomap/tags.js +101 -0
- package/dist/repomap/tags.js.map +1 -0
- package/dist/retrieve.d.ts +14 -0
- package/dist/retrieve.d.ts.map +1 -0
- package/dist/retrieve.js +44 -0
- package/dist/retrieve.js.map +1 -0
- package/dist/stages.d.ts +22 -0
- package/dist/stages.d.ts.map +1 -0
- package/dist/stages.js +33 -0
- package/dist/stages.js.map +1 -0
- package/dist/stats.d.ts +40 -0
- package/dist/stats.d.ts.map +1 -0
- package/dist/stats.js +37 -0
- package/dist/stats.js.map +1 -0
- package/dist/store-dir.d.ts +98 -0
- package/dist/store-dir.d.ts.map +1 -0
- package/dist/store-dir.js +379 -0
- package/dist/store-dir.js.map +1 -0
- package/dist/store.d.ts +36 -0
- package/dist/store.d.ts.map +1 -0
- package/dist/store.js +56 -0
- package/dist/store.js.map +1 -0
- package/dist/types.d.ts +298 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/grammars/tree-sitter-bash.wasm +0 -0
- package/grammars/tree-sitter-c.wasm +0 -0
- package/grammars/tree-sitter-c_sharp.wasm +0 -0
- package/grammars/tree-sitter-cpp.wasm +0 -0
- package/grammars/tree-sitter-go.wasm +0 -0
- package/grammars/tree-sitter-java.wasm +0 -0
- package/grammars/tree-sitter-javascript.wasm +0 -0
- package/grammars/tree-sitter-kotlin.wasm +0 -0
- package/grammars/tree-sitter-php.wasm +0 -0
- package/grammars/tree-sitter-python.wasm +0 -0
- package/grammars/tree-sitter-ruby.wasm +0 -0
- package/grammars/tree-sitter-rust.wasm +0 -0
- package/grammars/tree-sitter-swift.wasm +0 -0
- package/grammars/tree-sitter-tsx.wasm +0 -0
- package/grammars/tree-sitter-typescript.wasm +0 -0
- package/package.json +68 -0
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
/** Languages smelt can parse structurally. Everything else falls back to the lexical planner. */
|
|
2
|
+
export type LanguageId = 'typescript' | 'tsx' | 'javascript' | 'rust' | 'python' | 'go' | 'java' | 'c' | 'cpp' | 'c_sharp' | 'ruby' | 'php' | 'kotlin' | 'swift' | 'bash';
|
|
3
|
+
/** `'unknown'` is a first-class outcome, not a failure: it selects the lexical planner. */
|
|
4
|
+
export type DetectedLanguage = LanguageId | 'unknown';
|
|
5
|
+
/** A half-open byte range `[start, end)` into the UTF-8 bytes of the input. */
|
|
6
|
+
export interface ByteRange {
|
|
7
|
+
readonly start: number;
|
|
8
|
+
readonly end: number;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Why a range was elided, in two registers: a stable id for counters, and a sentence
|
|
12
|
+
* a human can read in a diff. Law 2 — every elision is explainable — lives here:
|
|
13
|
+
* if you cannot write the sentence, the rule does not ship.
|
|
14
|
+
*/
|
|
15
|
+
export interface ElisionReason {
|
|
16
|
+
/** Stable machine id, e.g. `'sibling-collapse'`, `'line-window'`. */
|
|
17
|
+
readonly rule: string;
|
|
18
|
+
/** e.g. `'collapsed 3 sibling functions'`. Present tense, no trailing period. */
|
|
19
|
+
readonly explanation: string;
|
|
20
|
+
}
|
|
21
|
+
/** One range a planner proposes to remove. Plans are pure data — inspectable and testable. */
|
|
22
|
+
export interface PlannedElision {
|
|
23
|
+
readonly range: ByteRange;
|
|
24
|
+
readonly reason: ElisionReason;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* The complete output of a planner: the whole decision, before anything is mutated.
|
|
28
|
+
* A plan can be logged, diffed, snapshot-tested, and rejected without touching the text.
|
|
29
|
+
*/
|
|
30
|
+
export interface ElisionPlan {
|
|
31
|
+
readonly planner: string;
|
|
32
|
+
readonly language: DetectedLanguage;
|
|
33
|
+
readonly elisions: readonly PlannedElision[];
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* The seam through which a planner asks what a marker will cost, in UTF-8 bytes.
|
|
37
|
+
*
|
|
38
|
+
* Marker cost is `apply.ts`'s fact — the applier renders the marker, so only the
|
|
39
|
+
* applier knows its price. Planners need that price for two decisions (profitability:
|
|
40
|
+
* a marker that costs more than it removes grows the output; and budget prediction:
|
|
41
|
+
* which ladder rung actually fits), and before this seam each planner rebuilt the
|
|
42
|
+
* marker machinery privately to measure it — correct, but an inversion. Now
|
|
43
|
+
* `markerPricing()` in `apply.ts` builds the one adapter from the exact builder
|
|
44
|
+
* `applyPlan` will use, and planners ask it. They never guess, and they never render
|
|
45
|
+
* a marker of their own.
|
|
46
|
+
*
|
|
47
|
+
* The price is exact, not an estimate: the cost of the marker this elision would
|
|
48
|
+
* earn, comment leader and all, with a stand-in hash of the real hash's length —
|
|
49
|
+
* marker cost depends on the hash's *length*, never its value.
|
|
50
|
+
*/
|
|
51
|
+
export interface MarkerPricing {
|
|
52
|
+
/** The exact UTF-8 byte cost of the marker an elision with this reason and size would earn. */
|
|
53
|
+
costBytes(reason: ElisionReason, elidedBytes: number): number;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* What the caller hands a planner.
|
|
57
|
+
*
|
|
58
|
+
* Constructed centrally: `createSmelter` (and through it, the CLI) builds the one
|
|
59
|
+
* `PlanInput` per call, including its {@link MarkerPricing} — a caller invoking
|
|
60
|
+
* `planLexical`/`planStructural` directly builds `pricing` with `markerPricing()`
|
|
61
|
+
* from `apply.ts`. A JS caller who omits it gets {@link MissingMarkerPricingError}
|
|
62
|
+
* at plan time, not a guessed cost.
|
|
63
|
+
*/
|
|
64
|
+
export interface PlanInput {
|
|
65
|
+
readonly text: string;
|
|
66
|
+
readonly language: DetectedLanguage;
|
|
67
|
+
/** Soft ceiling for the emitted output, in UTF-8 bytes. Planners aim under it. */
|
|
68
|
+
readonly budgetBytes: number;
|
|
69
|
+
/**
|
|
70
|
+
* What the caller was actually looking for — grep pattern, symbol name, error string.
|
|
71
|
+
* Planners keep matching regions and collapse around them.
|
|
72
|
+
*/
|
|
73
|
+
readonly focus?: readonly string[];
|
|
74
|
+
/**
|
|
75
|
+
* What a marker costs. Required: a planner that guesses marker cost can plan an
|
|
76
|
+
* elision that grows the output. See {@link MarkerPricing}; built by
|
|
77
|
+
* `markerPricing()` in `apply.ts` from the exact builder `applyPlan` will use.
|
|
78
|
+
*/
|
|
79
|
+
readonly pricing: MarkerPricing;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* A planner decides *what* to remove. It never removes anything itself; `applyPlan`
|
|
83
|
+
* does that. Keeping the decision and the mutation apart is what makes the decision
|
|
84
|
+
* testable in isolation.
|
|
85
|
+
*/
|
|
86
|
+
export interface Planner {
|
|
87
|
+
readonly id: string;
|
|
88
|
+
plan(input: PlanInput): Promise<ElisionPlan>;
|
|
89
|
+
}
|
|
90
|
+
/** One elision that actually happened, with the receipt needed to undo it. */
|
|
91
|
+
export interface AppliedElision {
|
|
92
|
+
/** Content hash of the removed bytes — the key `retrieve()` takes. */
|
|
93
|
+
readonly hash: string;
|
|
94
|
+
/** Where the removed bytes were in the *input*. */
|
|
95
|
+
readonly range: ByteRange;
|
|
96
|
+
/**
|
|
97
|
+
* Where the marker sits in the *output*. Law 3 — every elision is reversible — needs
|
|
98
|
+
* this: {@link Reconstructor} splices stored bytes back over these ranges. Without it,
|
|
99
|
+
* "reversible" would mean parsing markers back out of the text, which is a guess.
|
|
100
|
+
* This is a fact recorded at the moment of the cut.
|
|
101
|
+
*/
|
|
102
|
+
readonly outputRange: ByteRange;
|
|
103
|
+
/** Size of the removed content, in UTF-8 bytes. */
|
|
104
|
+
readonly bytes: number;
|
|
105
|
+
readonly reason: ElisionReason;
|
|
106
|
+
/** The exact marker text substituted into the output. */
|
|
107
|
+
readonly marker: string;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* A consumer-supplied counter, so a caller who bills in tokens can *see* tokens.
|
|
111
|
+
*
|
|
112
|
+
* Budgets in smelt's core are UTF-8 bytes, permanently — bytes are the only unit that
|
|
113
|
+
* is computable locally for every model, and they mean the same thing in five years.
|
|
114
|
+
* See `docs/HANDOFF.md` § "Decision 1". This hook does not change that: it adds a
|
|
115
|
+
* second, labelled number to the result. The plan is still made in bytes.
|
|
116
|
+
*
|
|
117
|
+
* Both `id` and `unit` are required, and that is a Law 4 decision rather than
|
|
118
|
+
* bookkeeping: a token count is meaningless without naming the tokenizer that produced
|
|
119
|
+
* it. Anthropic's own docs record that Claude 4.7 and later use a newer tokenizer where
|
|
120
|
+
* the same text yields roughly 30% more tokens than on earlier models — so `1,204
|
|
121
|
+
* tokens` is not a fact, and `1,204 tokens (claude-4.7/count_tokens)` is.
|
|
122
|
+
*
|
|
123
|
+
* **This hook does not relax Law 1.** smelt imports no transport and the guard proves
|
|
124
|
+
* that about smelt's own modules; it cannot prove it about a function you hand in. A
|
|
125
|
+
* `count()` that calls an API makes *your* process call an API, from a line in *your*
|
|
126
|
+
* source — exactly the arrangement {@link RerankStage} already describes. `count` is
|
|
127
|
+
* synchronous on purpose: local tokenizers are synchronous, and network clients are not.
|
|
128
|
+
*/
|
|
129
|
+
export interface Measure {
|
|
130
|
+
/** Names the counter, e.g. `'tiktoken/o200k_base'` or `'claude-4.7/count_tokens'`. */
|
|
131
|
+
readonly id: string;
|
|
132
|
+
/** The unit `count()` returns, e.g. `'tokens'`. Printed next to the number. */
|
|
133
|
+
readonly unit: string;
|
|
134
|
+
/** Local, synchronous count over the whole string. */
|
|
135
|
+
count(text: string): number;
|
|
136
|
+
}
|
|
137
|
+
/** A second size for a result, in someone else's unit, with the counter named. */
|
|
138
|
+
export interface MeasuredSize {
|
|
139
|
+
/** {@link Measure.id} of the counter that produced these numbers. */
|
|
140
|
+
readonly measure: string;
|
|
141
|
+
/** {@link Measure.unit}. */
|
|
142
|
+
readonly unit: string;
|
|
143
|
+
readonly input: number;
|
|
144
|
+
readonly output: number;
|
|
145
|
+
}
|
|
146
|
+
/** The result of smelting one blob of text. */
|
|
147
|
+
export interface SmeltResult {
|
|
148
|
+
readonly text: string;
|
|
149
|
+
readonly inputBytes: number;
|
|
150
|
+
readonly outputBytes: number;
|
|
151
|
+
readonly planner: string;
|
|
152
|
+
readonly language: DetectedLanguage;
|
|
153
|
+
readonly elisions: readonly AppliedElision[];
|
|
154
|
+
/** Present only when the caller supplied a {@link Measure}. Never invented. */
|
|
155
|
+
readonly measured?: MeasuredSize;
|
|
156
|
+
}
|
|
157
|
+
/** Options for a single `smelt()` call. */
|
|
158
|
+
export interface SmeltOptions {
|
|
159
|
+
/** Soft ceiling for the emitted output, in UTF-8 bytes. */
|
|
160
|
+
readonly budgetBytes: number;
|
|
161
|
+
/** Filename or path, used to detect the language. Optional; detection falls back to `'unknown'`. */
|
|
162
|
+
readonly path?: string;
|
|
163
|
+
/** Override language detection entirely. */
|
|
164
|
+
readonly language?: DetectedLanguage;
|
|
165
|
+
/** What the caller was looking for. See {@link PlanInput.focus}. */
|
|
166
|
+
readonly focus?: readonly string[];
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* Reversibility, as a callable. Takes a {@link SmeltResult} and the store that holds its
|
|
170
|
+
* elided bytes, and returns the original text — byte for byte.
|
|
171
|
+
*/
|
|
172
|
+
export type Reconstructor = (result: SmeltResult, store: ElisionStore) => string;
|
|
173
|
+
/**
|
|
174
|
+
* The numbers that keep smelt honest about itself.
|
|
175
|
+
*
|
|
176
|
+
* Law 3 says elisions are reversible. That is cheap to satisfy and easy to abuse: a
|
|
177
|
+
* compressor that cuts everything is "reversible" and useless. The *retrieve rate* is
|
|
178
|
+
* the tell. If the model keeps calling `smelt_retrieve`, smelt cut material the task
|
|
179
|
+
* needed, and the round trip cost more tokens than the elision saved.
|
|
180
|
+
*
|
|
181
|
+
* So: `expansionRate` is not telemetry. It is the metric a caller tunes budgets
|
|
182
|
+
* against, and the only number smelt is willing to have an opinion about — because it
|
|
183
|
+
* measures it locally, per session, on the caller's own traffic.
|
|
184
|
+
*/
|
|
185
|
+
export interface RetrieveStats {
|
|
186
|
+
/** Distinct blobs put into the store. */
|
|
187
|
+
readonly elisionsStored: number;
|
|
188
|
+
/** Total bytes held by the store. */
|
|
189
|
+
readonly bytesStored: number;
|
|
190
|
+
/** Every `retrieve()` call, including repeats and misses. */
|
|
191
|
+
readonly retrieveCalls: number;
|
|
192
|
+
/** Distinct hashes successfully retrieved at least once. */
|
|
193
|
+
readonly uniqueRetrieved: number;
|
|
194
|
+
/** Calls for a hash the store does not hold. Non-zero means a bug, not over-pruning. */
|
|
195
|
+
readonly misses: number;
|
|
196
|
+
/**
|
|
197
|
+
* `uniqueRetrieved / elisionsStored`, or `0` when nothing has been stored.
|
|
198
|
+
*
|
|
199
|
+
* Read it as: *what fraction of what smelt hid did the model have to ask for back?*
|
|
200
|
+
* There is no universally right value, and smelt does not ship a threshold it has
|
|
201
|
+
* not measured. Rising across a workload is the signal.
|
|
202
|
+
*/
|
|
203
|
+
readonly expansionRate: number;
|
|
204
|
+
/**
|
|
205
|
+
* The one degenerate outcome smelt is willing to name: **every distinct blob it hid
|
|
206
|
+
* was asked for again.**
|
|
207
|
+
*
|
|
208
|
+
* smelt ships no expansion-rate threshold, because a threshold is a policy claim it
|
|
209
|
+
* has no basis for and the right rate depends on how aggressive a budget the caller
|
|
210
|
+
* chose — and a library printing warnings into someone else's process is bad manners.
|
|
211
|
+
* This is not a threshold. At `uniqueRetrieved === elisionsStored` the elision
|
|
212
|
+
* achieved nothing and cost a round trip: an arithmetic fact, not a preference. What
|
|
213
|
+
* to do about it is the caller's call.
|
|
214
|
+
*
|
|
215
|
+
* `false` for an empty store — nothing was hidden, so nothing was defeated.
|
|
216
|
+
*/
|
|
217
|
+
readonly allElisionsRetrieved: boolean;
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* Local, content-addressed storage for elided bytes. No network, no eviction in v1 —
|
|
221
|
+
* evicting is how "reversible" quietly becomes "reversible for a while".
|
|
222
|
+
*/
|
|
223
|
+
export interface ElisionStore {
|
|
224
|
+
/** Store content, returning its hash. Idempotent for identical content. */
|
|
225
|
+
put(content: string): string;
|
|
226
|
+
/** The stored content, or `undefined` if this store never held that hash. */
|
|
227
|
+
peek(hash: string): string | undefined;
|
|
228
|
+
/**
|
|
229
|
+
* The stored content, *counted* as a retrieval. This is what the model's tool calls.
|
|
230
|
+
* @throws {UnknownHashError} when the hash is unknown.
|
|
231
|
+
*/
|
|
232
|
+
retrieve(hash: string): string;
|
|
233
|
+
has(hash: string): boolean;
|
|
234
|
+
/** A snapshot of the counters. See {@link RetrieveStats}. */
|
|
235
|
+
stats(): RetrieveStats;
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* The retrieval tool a consumer exposes to its model. Deliberately not an MCP or
|
|
239
|
+
* provider-specific shape — smelt does not know which SDK you use. The consumer adapts
|
|
240
|
+
* this into its own tool schema; the contract is `hash in, exact bytes out`.
|
|
241
|
+
*/
|
|
242
|
+
export interface RetrieveTool {
|
|
243
|
+
/** `'smelt_retrieve'`. Stable — consumers hard-code it in prompts. */
|
|
244
|
+
readonly name: string;
|
|
245
|
+
/** Prose the consumer can put straight into a tool description. */
|
|
246
|
+
readonly description: string;
|
|
247
|
+
/** JSON-Schema-shaped parameter description, for consumers that want one. */
|
|
248
|
+
readonly inputSchema: {
|
|
249
|
+
readonly type: 'object';
|
|
250
|
+
readonly properties: {
|
|
251
|
+
readonly hash: {
|
|
252
|
+
readonly type: 'string';
|
|
253
|
+
};
|
|
254
|
+
};
|
|
255
|
+
readonly required: readonly ['hash'];
|
|
256
|
+
};
|
|
257
|
+
/** @throws {UnknownHashError} when the hash is unknown. */
|
|
258
|
+
invoke(input: {
|
|
259
|
+
readonly hash: string;
|
|
260
|
+
}): string;
|
|
261
|
+
}
|
|
262
|
+
/** A candidate handed to a {@link RerankStage}: an opaque id plus the text to judge. */
|
|
263
|
+
export interface RerankCandidate {
|
|
264
|
+
readonly id: string;
|
|
265
|
+
readonly text: string;
|
|
266
|
+
}
|
|
267
|
+
/** A reranked candidate, most relevant first. `score` is the stage's own scale. */
|
|
268
|
+
export interface RerankedCandidate extends RerankCandidate {
|
|
269
|
+
readonly score: number;
|
|
270
|
+
}
|
|
271
|
+
/**
|
|
272
|
+
* Relevance reranking — a *seam*, not a feature.
|
|
273
|
+
*
|
|
274
|
+
* Hosted rerankers are good and smelt will never bundle one, because bundling would
|
|
275
|
+
* break Law 1: the moment smelt ships a default reranker, `smelt()` can make a network
|
|
276
|
+
* call that the caller did not ask for and cannot see. A consumer that wants one
|
|
277
|
+
* implements this interface, wires its own key, and owns the fact that its context now
|
|
278
|
+
* leaves the machine. That decision must be legible in the consumer's own source.
|
|
279
|
+
*/
|
|
280
|
+
export interface RerankStage {
|
|
281
|
+
readonly id: string;
|
|
282
|
+
/** May make network calls — that is the consumer's choice, made in the consumer's code. */
|
|
283
|
+
rerank(candidates: readonly RerankCandidate[], query: string): Promise<readonly RerankedCandidate[]>;
|
|
284
|
+
}
|
|
285
|
+
/**
|
|
286
|
+
* Learned distillation — rewriting content with a model instead of cutting it.
|
|
287
|
+
*
|
|
288
|
+
* Out of v1 for a reason beyond the network: a distilled paragraph cannot satisfy
|
|
289
|
+
* Law 2. "The model summarised this" is not an explanation of what was removed, and
|
|
290
|
+
* the removed material is no longer recoverable from the output. If this ever ships,
|
|
291
|
+
* it ships as a stage that stores the original and explains itself in the same terms
|
|
292
|
+
* every other rule does.
|
|
293
|
+
*/
|
|
294
|
+
export interface DistillStage {
|
|
295
|
+
readonly id: string;
|
|
296
|
+
distill(text: string, budgetBytes: number): Promise<string>;
|
|
297
|
+
}
|
|
298
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,iGAAiG;AACjG,MAAM,MAAM,UAAU,GAClB,YAAY,GACZ,KAAK,GACL,YAAY,GACZ,MAAM,GACN,QAAQ,GACR,IAAI,GACJ,MAAM,GACN,GAAG,GACH,KAAK,GACL,SAAS,GACT,MAAM,GACN,KAAK,GACL,QAAQ,GACR,OAAO,GACP,MAAM,CAAC;AAEX,2FAA2F;AAC3F,MAAM,MAAM,gBAAgB,GAAG,UAAU,GAAG,SAAS,CAAC;AAEtD,+EAA+E;AAC/E,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;CACtB;AAED;;;;GAIG;AACH,MAAM,WAAW,aAAa;IAC5B,qEAAqE;IACrE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,iFAAiF;IACjF,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED,8FAA8F;AAC9F,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;CAChC;AAED;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,gBAAgB,CAAC;IACpC,QAAQ,CAAC,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAC;CAC9C;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,aAAa;IAC5B,+FAA+F;IAC/F,SAAS,CAAC,MAAM,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,GAAG,MAAM,CAAC;CAC/D;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,gBAAgB,CAAC;IACpC,kFAAkF;IAClF,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B;;;OAGG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC;;;;OAIG;IACH,QAAQ,CAAC,OAAO,EAAE,aAAa,CAAC;CACjC;AAED;;;;GAIG;AACH,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;CAC9C;AAED,8EAA8E;AAC9E,MAAM,WAAW,cAAc;IAC7B,sEAAsE;IACtE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,mDAAmD;IACnD,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;IAC1B;;;;;OAKG;IACH,QAAQ,CAAC,WAAW,EAAE,SAAS,CAAC;IAChC,mDAAmD;IACnD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC;IAC/B,yDAAyD;IACzD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,WAAW,OAAO;IACtB,sFAAsF;IACtF,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,+EAA+E;IAC/E,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,sDAAsD;IACtD,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;CAC7B;AAED,kFAAkF;AAClF,MAAM,WAAW,YAAY;IAC3B,qEAAqE;IACrE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,4BAA4B;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,+CAA+C;AAC/C,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,gBAAgB,CAAC;IACpC,QAAQ,CAAC,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAC;IAC7C,+EAA+E;IAC/E,QAAQ,CAAC,QAAQ,CAAC,EAAE,YAAY,CAAC;CAClC;AAED,2CAA2C;AAC3C,MAAM,WAAW,YAAY;IAC3B,2DAA2D;IAC3D,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,oGAAoG;IACpG,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,4CAA4C;IAC5C,QAAQ,CAAC,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IACrC,oEAAoE;IACpE,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACpC;AAED;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,KAAK,MAAM,CAAC;AAMjF;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,aAAa;IAC5B,yCAAyC;IACzC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,qCAAqC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,6DAA6D;IAC7D,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,4DAA4D;IAC5D,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,wFAAwF;IACxF,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;;;;;;OAMG;IACH,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B;;;;;;;;;;;;OAYG;IACH,QAAQ,CAAC,oBAAoB,EAAE,OAAO,CAAC;CACxC;AAED;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B,2EAA2E;IAC3E,GAAG,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC;IAC7B,6EAA6E;IAC7E,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;IACvC;;;OAGG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IAC/B,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;IAC3B,6DAA6D;IAC7D,KAAK,IAAI,aAAa,CAAC;CACxB;AAED;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,sEAAsE;IACtE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,mEAAmE;IACnE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,6EAA6E;IAC7E,QAAQ,CAAC,WAAW,EAAE;QACpB,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;QACxB,QAAQ,CAAC,UAAU,EAAE;YAAE,QAAQ,CAAC,IAAI,EAAE;gBAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAA;aAAE,CAAA;SAAE,CAAC;QACpE,QAAQ,CAAC,QAAQ,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;KACtC,CAAC;IACF,2DAA2D;IAC3D,MAAM,CAAC,KAAK,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,CAAC;CAClD;AAMD,wFAAwF;AACxF,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED,mFAAmF;AACnF,MAAM,WAAW,iBAAkB,SAAQ,eAAe;IACxD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,2FAA2F;IAC3F,MAAM,CACJ,UAAU,EAAE,SAAS,eAAe,EAAE,EACtC,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,SAAS,iBAAiB,EAAE,CAAC,CAAC;CAC1C;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC7D"}
|
package/dist/types.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
package/package.json
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@smeltjs/core",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Structure-aware, reversible, offline context optimization for AI coding agents.",
|
|
5
|
+
"license": "Apache-2.0",
|
|
6
|
+
"author": "mong-x",
|
|
7
|
+
"homepage": "https://github.com/mong-x/smelt#readme",
|
|
8
|
+
"repository": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"url": "git+https://github.com/mong-x/smelt.git",
|
|
11
|
+
"directory": "packages/core"
|
|
12
|
+
},
|
|
13
|
+
"bugs": "https://github.com/mong-x/smelt/issues",
|
|
14
|
+
"keywords": [
|
|
15
|
+
"llm",
|
|
16
|
+
"context",
|
|
17
|
+
"tokens",
|
|
18
|
+
"tree-sitter",
|
|
19
|
+
"agents",
|
|
20
|
+
"compression",
|
|
21
|
+
"prompt-cache",
|
|
22
|
+
"offline"
|
|
23
|
+
],
|
|
24
|
+
"type": "module",
|
|
25
|
+
"sideEffects": false,
|
|
26
|
+
"engines": {
|
|
27
|
+
"node": "^20.19.0 || >=22.12.0"
|
|
28
|
+
},
|
|
29
|
+
"files": [
|
|
30
|
+
"dist",
|
|
31
|
+
"grammars",
|
|
32
|
+
"README.md",
|
|
33
|
+
"THIRD-PARTY.md"
|
|
34
|
+
],
|
|
35
|
+
"exports": {
|
|
36
|
+
".": {
|
|
37
|
+
"types": "./dist/index.d.ts",
|
|
38
|
+
"import": "./dist/index.js",
|
|
39
|
+
"default": "./dist/index.js"
|
|
40
|
+
},
|
|
41
|
+
"./package.json": "./package.json"
|
|
42
|
+
},
|
|
43
|
+
"bin": {
|
|
44
|
+
"smelt": "dist/cli/bin.js"
|
|
45
|
+
},
|
|
46
|
+
"main": "./dist/index.js",
|
|
47
|
+
"types": "./dist/index.d.ts",
|
|
48
|
+
"publishConfig": {
|
|
49
|
+
"access": "public"
|
|
50
|
+
},
|
|
51
|
+
"scripts": {
|
|
52
|
+
"build": "tsc -p tsconfig.build.json && node ../../scripts/bundle-grammars.mjs",
|
|
53
|
+
"typecheck": "tsc -p tsconfig.json --noEmit",
|
|
54
|
+
"test": "vitest run",
|
|
55
|
+
"test:watch": "vitest",
|
|
56
|
+
"prepack": "pnpm run build && node ../../scripts/generate-third-party.mjs",
|
|
57
|
+
"bench": "node bench/run.mjs"
|
|
58
|
+
},
|
|
59
|
+
"dependencies": {
|
|
60
|
+
"web-tree-sitter": "~0.25.10"
|
|
61
|
+
},
|
|
62
|
+
"devDependencies": {
|
|
63
|
+
"@types/node": "^26.4.0",
|
|
64
|
+
"tree-sitter-wasms": "^0.1.13",
|
|
65
|
+
"typescript": "^7.0.2",
|
|
66
|
+
"vitest": "^4.1.11"
|
|
67
|
+
}
|
|
68
|
+
}
|