@giovannijecha/jecode 0.8.2 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -278
- package/dist/accounts.js +17 -13
- package/dist/batch.js +54 -6
- package/dist/context/budget.js +13 -1
- package/dist/context/compactor.js +5 -4
- package/dist/context/estimate.js +43 -1
- package/dist/context/manual.js +8 -4
- package/dist/context/policy.js +68 -18
- package/dist/controller-request.js +8 -8
- package/dist/controller.js +6 -1
- package/dist/conversation.js +94 -33
- package/dist/credential-safety.js +56 -9
- package/dist/credentials.js +32 -4
- package/dist/input-boundary.js +80 -0
- package/dist/main.js +4 -1
- package/dist/openai-oauth-callback.js +1 -1
- package/dist/process-shutdown.js +52 -0
- package/dist/providers/anthropic-stream.js +24 -20
- package/dist/providers/anthropic-wire.js +7 -2
- package/dist/providers/ollama-wire.js +7 -15
- package/dist/providers/ollama.js +1 -0
- package/dist/providers/openai-wire.js +2 -16
- package/dist/providers/tool-input.js +17 -0
- package/dist/sessions/codec.js +2 -1
- package/dist/sessions/lease.js +7 -0
- package/dist/sessions/runtime.js +11 -3
- package/dist/sessions/store.js +70 -21
- package/dist/settings.js +10 -5
- package/dist/start.js +12 -2
- package/dist/text-boundary.js +2 -0
- package/dist/tui/app-input.js +40 -5
- package/dist/tui/app-state.js +1 -0
- package/dist/tui/app-workflows.js +1 -0
- package/dist/tui/app.js +11 -3
- package/dist/tui/editor.js +2 -0
- package/dist/tui/keys.js +64 -5
- package/dist/tui/overlay.js +12 -4
- package/dist/tui/picker.js +2 -0
- package/dist/tui/screen.js +5 -17
- package/dist/user-store.js +54 -0
- package/package.json +6 -3
- /package/{docs/assets/brand → assets}/jeco-256.png +0 -0
package/dist/context/manual.js
CHANGED
|
@@ -6,14 +6,15 @@
|
|
|
6
6
|
import { recordAuxiliaryUsage } from "../usage.js";
|
|
7
7
|
import { resolveContextPolicy } from "./capacity.js";
|
|
8
8
|
import { compactContext } from "./compactor.js";
|
|
9
|
-
import {
|
|
9
|
+
import { estimateTokensResponsive, planCompaction } from "./policy.js";
|
|
10
10
|
const MIN_PREFIX_TOKENS = 512;
|
|
11
11
|
export async function compactSession(session, options = {}) {
|
|
12
12
|
const active = session.conversation.activeNode;
|
|
13
13
|
if (active === undefined)
|
|
14
14
|
return "unchanged";
|
|
15
15
|
const context = session.conversation.contextHistory;
|
|
16
|
-
|
|
16
|
+
const estimatedInputTokens = await estimateTokensResponsive(context, options.signal);
|
|
17
|
+
if (estimatedInputTokens < MIN_PREFIX_TOKENS)
|
|
17
18
|
return "unchanged";
|
|
18
19
|
if (session.conversation.nodes.some((node) => node.parentId === active.id)) {
|
|
19
20
|
throw new Error("continue this branch before compacting");
|
|
@@ -29,8 +30,9 @@ export async function compactSession(session, options = {}) {
|
|
|
29
30
|
const coveredMessages = active.context?.throughNodeId === active.id
|
|
30
31
|
? active.context.messageCount
|
|
31
32
|
: 0;
|
|
32
|
-
const plan = planCompaction(context, active.messages, coveredMessages, session.usage.lastInputTokens, true, policy);
|
|
33
|
-
if (plan === undefined ||
|
|
33
|
+
const plan = await planCompaction(context, active.messages, coveredMessages, session.usage.lastInputTokens, true, policy, estimatedInputTokens, options.signal);
|
|
34
|
+
if (plan === undefined ||
|
|
35
|
+
await estimateTokensResponsive(plan.prefix, options.signal) < MIN_PREFIX_TOKENS) {
|
|
34
36
|
options.onStatus?.();
|
|
35
37
|
return "unchanged";
|
|
36
38
|
}
|
|
@@ -43,6 +45,8 @@ export async function compactSession(session, options = {}) {
|
|
|
43
45
|
nodeId: active.id,
|
|
44
46
|
coveredMessages,
|
|
45
47
|
lastInputTokens: session.usage.lastInputTokens,
|
|
48
|
+
estimatedInputTokens,
|
|
49
|
+
precomputedPlan: plan,
|
|
46
50
|
signal: options.signal,
|
|
47
51
|
force: true,
|
|
48
52
|
failLoudly: true,
|
package/dist/context/policy.js
CHANGED
|
@@ -1,15 +1,18 @@
|
|
|
1
1
|
// Provider-neutral context pressure and safe compaction boundaries.
|
|
2
|
-
import { estimateSerializedTokens } from "./estimate.js";
|
|
2
|
+
import { estimateSerializedTokens, estimateSerializedTokensResponsive, } from "./estimate.js";
|
|
3
3
|
export const DEFAULT_COMPACTION_PERCENT = 85;
|
|
4
4
|
export const MIN_COMPACTION_PERCENT = 50;
|
|
5
5
|
export const MAX_COMPACTION_PERCENT = 95;
|
|
6
6
|
export const FALLBACK_CONTEXT_WINDOW_TOKENS = 200_000;
|
|
7
7
|
export const REQUEST_ESTIMATE_HEADROOM_PERCENT = 5;
|
|
8
8
|
export const MIN_REQUEST_OUTPUT_TOKENS = 256;
|
|
9
|
-
export function planCompaction(context, turn, coveredMessages, lastInputTokens, force, policy) {
|
|
9
|
+
export async function planCompaction(context, turn, coveredMessages, lastInputTokens, force, policy, estimatedInputTokens, signal) {
|
|
10
10
|
if (!validPolicy(policy) || coveredMessages < 0 || coveredMessages > turn.length)
|
|
11
11
|
return undefined;
|
|
12
|
-
|
|
12
|
+
if (estimatedInputTokens !== undefined &&
|
|
13
|
+
(!Number.isSafeInteger(estimatedInputTokens) || estimatedInputTokens <= 0))
|
|
14
|
+
return undefined;
|
|
15
|
+
const estimated = estimatedInputTokens ?? await estimateTokensResponsive(context, signal);
|
|
13
16
|
if (!force &&
|
|
14
17
|
(estimated < policy.targetTokens || Math.max(estimated, lastInputTokens) < policy.triggerTokens))
|
|
15
18
|
return undefined;
|
|
@@ -17,15 +20,16 @@ export function planCompaction(context, turn, coveredMessages, lastInputTokens,
|
|
|
17
20
|
const contextPrefix = context.length - currentSuffix;
|
|
18
21
|
if (contextPrefix < 0)
|
|
19
22
|
return undefined;
|
|
20
|
-
if (!sameMessages(context.slice(contextPrefix), turn.slice(coveredMessages)))
|
|
23
|
+
if (!await sameMessages(context.slice(contextPrefix), turn.slice(coveredMessages), signal))
|
|
21
24
|
return undefined;
|
|
22
25
|
const targetTokens = force
|
|
23
26
|
? Math.min(policy.targetTokens, Math.max(512, Math.floor(estimated / 4)))
|
|
24
27
|
: policy.targetTokens;
|
|
25
28
|
const recentTokens = Math.min(policy.recentTokens, Math.max(256, Math.floor(targetTokens / 2)));
|
|
26
|
-
|
|
29
|
+
const recent = await recentBoundary(turn, coveredMessages, recentTokens, signal);
|
|
30
|
+
let boundary = recent.boundary;
|
|
27
31
|
let tail = turn.slice(boundary);
|
|
28
|
-
if (turn.length > 1 &&
|
|
32
|
+
if (turn.length > 1 && recent.tokens > targetTokens) {
|
|
29
33
|
boundary = turn.length;
|
|
30
34
|
tail = [];
|
|
31
35
|
}
|
|
@@ -33,8 +37,9 @@ export function planCompaction(context, turn, coveredMessages, lastInputTokens,
|
|
|
33
37
|
if (prefixEnd <= 0 || prefixEnd > context.length)
|
|
34
38
|
return undefined;
|
|
35
39
|
const prefix = context.slice(0, prefixEnd);
|
|
36
|
-
if (!force &&
|
|
40
|
+
if (!force && await estimateTokensResponsive(prefix, signal) < policy.minimumPrefixTokens) {
|
|
37
41
|
return undefined;
|
|
42
|
+
}
|
|
38
43
|
return {
|
|
39
44
|
prefix: clone(prefix),
|
|
40
45
|
tail: clone(tail),
|
|
@@ -70,6 +75,9 @@ export function policyForContextWindow(context, compactionPercent) {
|
|
|
70
75
|
export function estimateTokens(messages) {
|
|
71
76
|
return estimateSerializedTokens(messages) + messages.length * 8;
|
|
72
77
|
}
|
|
78
|
+
export async function estimateTokensResponsive(messages, signal) {
|
|
79
|
+
return await estimateSerializedTokensResponsive(messages, signal) + messages.length * 8;
|
|
80
|
+
}
|
|
73
81
|
export function isContextOverflow(error) {
|
|
74
82
|
const candidate = error;
|
|
75
83
|
if (candidate.status !== 400 && candidate.status !== 413)
|
|
@@ -77,16 +85,45 @@ export function isContextOverflow(error) {
|
|
|
77
85
|
const detail = `${candidate.message}\n${candidate.body ?? ""}`;
|
|
78
86
|
return /context_length_exceeded|maximum context length|context window|prompt is too long|input (?:is )?too (?:long|large)|(?:input|prompt|context).{0,80}(?:exceed|maximum|max tokens)/i.test(detail);
|
|
79
87
|
}
|
|
80
|
-
function recentBoundary(turn, coveredMessages, recentTokens) {
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
+
async function recentBoundary(turn, coveredMessages, recentTokens, signal) {
|
|
89
|
+
const minimum = Math.max(coveredMessages, minimumRecentBoundary(turn));
|
|
90
|
+
const candidates = [];
|
|
91
|
+
for (let candidate = coveredMessages; candidate <= minimum; candidate++) {
|
|
92
|
+
if (candidate === minimum || safeBoundary(turn, candidate))
|
|
93
|
+
candidates.push(candidate);
|
|
94
|
+
}
|
|
95
|
+
const cache = new Map();
|
|
96
|
+
const estimateAt = (candidate) => {
|
|
97
|
+
let estimate = cache.get(candidate);
|
|
98
|
+
if (estimate === undefined) {
|
|
99
|
+
estimate = estimateTokensResponsive(turn.slice(candidate), signal);
|
|
100
|
+
cache.set(candidate, estimate);
|
|
101
|
+
}
|
|
102
|
+
return estimate;
|
|
103
|
+
};
|
|
104
|
+
const last = candidates.length - 1;
|
|
105
|
+
const minimumTokens = await estimateAt(candidates[last]);
|
|
106
|
+
if (minimumTokens > recentTokens) {
|
|
107
|
+
return { boundary: candidates[last], tokens: minimumTokens };
|
|
88
108
|
}
|
|
89
|
-
|
|
109
|
+
// Adding older messages raises the byte and literal floors. A lower-bound
|
|
110
|
+
// search therefore replaces the former serialization of every suffix.
|
|
111
|
+
let low = 0;
|
|
112
|
+
let high = last;
|
|
113
|
+
let selected = last;
|
|
114
|
+
while (low <= high) {
|
|
115
|
+
const middle = Math.floor((low + high) / 2);
|
|
116
|
+
const tokens = await estimateAt(candidates[middle]);
|
|
117
|
+
if (tokens <= recentTokens) {
|
|
118
|
+
selected = middle;
|
|
119
|
+
high = middle - 1;
|
|
120
|
+
}
|
|
121
|
+
else {
|
|
122
|
+
low = middle + 1;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
const boundary = candidates[selected];
|
|
126
|
+
return { boundary, tokens: await estimateAt(boundary) };
|
|
90
127
|
}
|
|
91
128
|
function minimumRecentBoundary(turn) {
|
|
92
129
|
const last = turn.at(-1);
|
|
@@ -127,8 +164,21 @@ function validPercent(value) {
|
|
|
127
164
|
return Number.isSafeInteger(value) &&
|
|
128
165
|
value >= MIN_COMPACTION_PERCENT && value <= MAX_COMPACTION_PERCENT;
|
|
129
166
|
}
|
|
130
|
-
function sameMessages(left, right) {
|
|
131
|
-
|
|
167
|
+
async function sameMessages(left, right, signal) {
|
|
168
|
+
if (left.length !== right.length)
|
|
169
|
+
return false;
|
|
170
|
+
for (let index = 0; index < left.length; index++) {
|
|
171
|
+
throwIfAborted(signal);
|
|
172
|
+
if (JSON.stringify(left[index]) !== JSON.stringify(right[index]))
|
|
173
|
+
return false;
|
|
174
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
175
|
+
}
|
|
176
|
+
return true;
|
|
177
|
+
}
|
|
178
|
+
function throwIfAborted(signal) {
|
|
179
|
+
if (signal?.aborted !== true)
|
|
180
|
+
return;
|
|
181
|
+
throw signal.reason instanceof Error ? signal.reason : new Error("aborted");
|
|
132
182
|
}
|
|
133
183
|
function clone(value) {
|
|
134
184
|
return structuredClone(value);
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
// One streamed provider request with a single safe context-overflow recovery.
|
|
2
|
-
import { budgetRequestFromInputTokens,
|
|
2
|
+
import { budgetRequestFromInputTokens, estimateRequestInputTokensResponsive, } from "./context/budget.js";
|
|
3
3
|
import { isContextOverflow } from "./context/policy.js";
|
|
4
4
|
export async function requestAssistant(history, current, specs, options, events, signal) {
|
|
5
5
|
let policy = await options.contextPolicy();
|
|
6
|
-
const prepared = await prepareContext(history, current, specs, options, events, policy, "budget");
|
|
6
|
+
const prepared = await prepareContext(history, current, specs, options, events, policy, "budget", signal);
|
|
7
7
|
let context = prepared.projected === undefined ? [...current] : clone(prepared.projected);
|
|
8
8
|
let inputTokens = prepared.inputTokens;
|
|
9
9
|
let recovered = false;
|
|
@@ -28,7 +28,7 @@ export async function requestAssistant(history, current, specs, options, events,
|
|
|
28
28
|
throw error;
|
|
29
29
|
if (isContextOverflow(error))
|
|
30
30
|
policy = await options.contextPolicy();
|
|
31
|
-
const next = await prepareContext(history, context, specs, options, events, policy, "overflow", error);
|
|
31
|
+
const next = await prepareContext(history, context, specs, options, events, policy, "overflow", signal, error, inputTokens);
|
|
32
32
|
if (next.projected === undefined)
|
|
33
33
|
throw error;
|
|
34
34
|
context = clone(next.projected);
|
|
@@ -37,12 +37,12 @@ export async function requestAssistant(history, current, specs, options, events,
|
|
|
37
37
|
}
|
|
38
38
|
}
|
|
39
39
|
}
|
|
40
|
-
async function prepareContext(history, context, specs, options, events, policy, reason, error) {
|
|
41
|
-
const inputTokens =
|
|
40
|
+
async function prepareContext(history, context, specs, options, events, policy, reason, signal, error, knownInputTokens) {
|
|
41
|
+
const inputTokens = knownInputTokens ?? await estimateRequestInputTokensResponsive({
|
|
42
42
|
system: options.system,
|
|
43
43
|
messages: context,
|
|
44
44
|
tools: specs,
|
|
45
|
-
});
|
|
45
|
+
}, signal);
|
|
46
46
|
const projected = await events.onContext?.(history, context, {
|
|
47
47
|
reason,
|
|
48
48
|
policy,
|
|
@@ -53,11 +53,11 @@ async function prepareContext(history, context, specs, options, events, policy,
|
|
|
53
53
|
projected,
|
|
54
54
|
inputTokens: projected === undefined
|
|
55
55
|
? inputTokens
|
|
56
|
-
:
|
|
56
|
+
: await estimateRequestInputTokensResponsive({
|
|
57
57
|
system: options.system,
|
|
58
58
|
messages: projected,
|
|
59
59
|
tools: specs,
|
|
60
|
-
}),
|
|
60
|
+
}, signal),
|
|
61
61
|
};
|
|
62
62
|
}
|
|
63
63
|
function clone(messages) {
|
package/dist/controller.js
CHANGED
|
@@ -185,11 +185,14 @@ function assertToolCallIds(calls) {
|
|
|
185
185
|
}
|
|
186
186
|
}
|
|
187
187
|
async function settle(call, current, total, options, events, signal, preview) {
|
|
188
|
+
throwIfAborted(signal);
|
|
189
|
+
if (call.inputError !== undefined) {
|
|
190
|
+
return refuse(call, call.inputError, "invalid arguments");
|
|
191
|
+
}
|
|
188
192
|
const tool = findTool(options.tools, call.name);
|
|
189
193
|
if (tool === undefined) {
|
|
190
194
|
return refuse(call, `no such tool: ${call.name}`, "unknown tool");
|
|
191
195
|
}
|
|
192
|
-
throwIfAborted(signal);
|
|
193
196
|
const approved = !tool.dangerous || await events.approve(call);
|
|
194
197
|
throwIfAborted(signal);
|
|
195
198
|
if (!approved) {
|
|
@@ -218,6 +221,8 @@ function refuse(call, reason, summary) {
|
|
|
218
221
|
* turn depends on it, because it exists for the user, not for the model.
|
|
219
222
|
*/
|
|
220
223
|
async function look(call, options, signal) {
|
|
224
|
+
if (call.inputError !== undefined)
|
|
225
|
+
return undefined;
|
|
221
226
|
const tool = findTool(options.tools, call.name);
|
|
222
227
|
if (tool?.preview === undefined)
|
|
223
228
|
return undefined;
|
package/dist/conversation.js
CHANGED
|
@@ -16,13 +16,17 @@ export const CONVERSATION_LIMITS = Object.freeze({
|
|
|
16
16
|
export class ConversationTree {
|
|
17
17
|
#nodes;
|
|
18
18
|
#activeNodeId;
|
|
19
|
-
|
|
20
|
-
|
|
19
|
+
#nodeBounds;
|
|
20
|
+
#bounds;
|
|
21
|
+
constructor(nodes, activeNodeId, nodeBounds, bounds) {
|
|
22
|
+
this.#nodes = nodes;
|
|
21
23
|
this.#activeNodeId = activeNodeId;
|
|
24
|
+
this.#nodeBounds = nodeBounds;
|
|
25
|
+
this.#bounds = bounds;
|
|
22
26
|
Object.freeze(this);
|
|
23
27
|
}
|
|
24
28
|
static empty() {
|
|
25
|
-
return new ConversationTree([], 0);
|
|
29
|
+
return new ConversationTree(Object.freeze([]), 0, Object.freeze([]), emptyBounds());
|
|
26
30
|
}
|
|
27
31
|
static restore(nodes, activeNodeId) {
|
|
28
32
|
if (nodes.length > CONVERSATION_LIMITS.nodes) {
|
|
@@ -39,11 +43,13 @@ export class ConversationTree {
|
|
|
39
43
|
assertPersistableNode(owned);
|
|
40
44
|
restored.push(owned);
|
|
41
45
|
}
|
|
42
|
-
|
|
46
|
+
const measured = measureBounds(restored);
|
|
47
|
+
assertBounds(measured.total);
|
|
48
|
+
assertContextPaths(restored, restored.filter((node) => node.context !== undefined));
|
|
43
49
|
if (!validNodeId(activeNodeId) ||
|
|
44
50
|
(activeNodeId !== 0 && restored[activeNodeId - 1]?.id !== activeNodeId))
|
|
45
51
|
throw new Error("conversation node does not exist");
|
|
46
|
-
return new ConversationTree(restored, activeNodeId);
|
|
52
|
+
return new ConversationTree(Object.freeze(restored), activeNodeId, Object.freeze(measured.nodes), measured.total);
|
|
47
53
|
}
|
|
48
54
|
/** Commit or extend the one prospective leaf turn. */
|
|
49
55
|
commit(draft, settlement) {
|
|
@@ -60,7 +66,7 @@ export class ConversationTree {
|
|
|
60
66
|
if (!validNodeId(nodeId) || (nodeId !== 0 && this.#nodes[nodeId - 1]?.id !== nodeId)) {
|
|
61
67
|
throw new Error("conversation node does not exist");
|
|
62
68
|
}
|
|
63
|
-
return new ConversationTree(this.#nodes, nodeId);
|
|
69
|
+
return new ConversationTree(this.#nodes, nodeId, this.#nodeBounds, this.#bounds);
|
|
64
70
|
}
|
|
65
71
|
node(nodeId) {
|
|
66
72
|
return nodeId === 0 ? undefined : this.#nodes[nodeId - 1];
|
|
@@ -132,9 +138,11 @@ export class ConversationTree {
|
|
|
132
138
|
});
|
|
133
139
|
assertTurn(node);
|
|
134
140
|
assertPersistableNode(node);
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
141
|
+
assertContextPath(this.#nodes, node);
|
|
142
|
+
const nodeBounds = measureNode(node);
|
|
143
|
+
const bounds = addBounds(this.#bounds, nodeBounds);
|
|
144
|
+
assertBounds(bounds);
|
|
145
|
+
return new ConversationTree(Object.freeze([...this.#nodes, node]), node.id, Object.freeze([...this.#nodeBounds, nodeBounds]), bounds);
|
|
138
146
|
}
|
|
139
147
|
#replace(draft, settlement) {
|
|
140
148
|
const id = draft.nodeId;
|
|
@@ -154,10 +162,16 @@ export class ConversationTree {
|
|
|
154
162
|
});
|
|
155
163
|
assertTurn(node);
|
|
156
164
|
assertPersistableNode(node);
|
|
165
|
+
assertContextPath(this.#nodes, node);
|
|
157
166
|
const nodes = [...this.#nodes];
|
|
158
167
|
nodes[id - 1] = node;
|
|
159
|
-
|
|
160
|
-
|
|
168
|
+
const priorBounds = this.#nodeBounds[id - 1];
|
|
169
|
+
const nextNodeBounds = measureNode(node);
|
|
170
|
+
const nodeBounds = [...this.#nodeBounds];
|
|
171
|
+
nodeBounds[id - 1] = nextNodeBounds;
|
|
172
|
+
const bounds = replaceBounds(this.#bounds, priorBounds, nextNodeBounds);
|
|
173
|
+
assertBounds(bounds);
|
|
174
|
+
return new ConversationTree(Object.freeze(nodes), id, Object.freeze(nodeBounds), bounds);
|
|
161
175
|
}
|
|
162
176
|
#path() {
|
|
163
177
|
const path = [];
|
|
@@ -174,13 +188,13 @@ export class ConversationTree {
|
|
|
174
188
|
}
|
|
175
189
|
}
|
|
176
190
|
function ownedNode(node) {
|
|
177
|
-
return
|
|
191
|
+
return deepFreeze({
|
|
178
192
|
...node,
|
|
179
|
-
identity:
|
|
180
|
-
messages:
|
|
181
|
-
blocks:
|
|
182
|
-
...(node.context === undefined ? {} : { context:
|
|
183
|
-
...(node.failure === undefined ? {} : { failure:
|
|
193
|
+
identity: { ...node.identity },
|
|
194
|
+
messages: clone(node.messages),
|
|
195
|
+
blocks: clone(node.blocks),
|
|
196
|
+
...(node.context === undefined ? {} : { context: { ...node.context } }),
|
|
197
|
+
...(node.failure === undefined ? {} : { failure: { ...node.failure } }),
|
|
184
198
|
});
|
|
185
199
|
}
|
|
186
200
|
function settledBlocks(blocks) {
|
|
@@ -225,28 +239,64 @@ function validSettlement(value) {
|
|
|
225
239
|
return value === "checkpointed" || value === "completed" ||
|
|
226
240
|
value === "failed" || value === "interrupted";
|
|
227
241
|
}
|
|
228
|
-
function assertBounds(
|
|
229
|
-
|
|
230
|
-
let transcriptCodeUnits = 0;
|
|
231
|
-
let contextCodeUnits = 0;
|
|
232
|
-
const contextOwners = [];
|
|
233
|
-
for (const node of nodes) {
|
|
234
|
-
messageCodeUnits += JSON.stringify(node.messages).length;
|
|
235
|
-
transcriptCodeUnits += JSON.stringify(node.blocks).length;
|
|
236
|
-
contextCodeUnits += node.context?.summary.length ?? 0;
|
|
237
|
-
if (node.context !== undefined)
|
|
238
|
-
contextOwners.push(node);
|
|
239
|
-
}
|
|
240
|
-
if (messageCodeUnits > CONVERSATION_LIMITS.messageCodeUnits) {
|
|
242
|
+
function assertBounds(bounds) {
|
|
243
|
+
if (bounds.messageCodeUnits > CONVERSATION_LIMITS.messageCodeUnits) {
|
|
241
244
|
throw new Error("conversation model history reached its session limit — start /new");
|
|
242
245
|
}
|
|
243
|
-
if (transcriptCodeUnits > CONVERSATION_LIMITS.transcriptCodeUnits) {
|
|
246
|
+
if (bounds.transcriptCodeUnits > CONVERSATION_LIMITS.transcriptCodeUnits) {
|
|
244
247
|
throw new Error("conversation transcript reached its session limit — start /new");
|
|
245
248
|
}
|
|
246
|
-
if (contextCodeUnits > CONVERSATION_LIMITS.contextCodeUnits) {
|
|
249
|
+
if (bounds.contextCodeUnits > CONVERSATION_LIMITS.contextCodeUnits) {
|
|
247
250
|
throw new Error("conversation context summaries reached their session limit — start /new");
|
|
248
251
|
}
|
|
249
|
-
|
|
252
|
+
}
|
|
253
|
+
function measureBounds(nodes) {
|
|
254
|
+
const measured = nodes.map(measureNode);
|
|
255
|
+
return {
|
|
256
|
+
nodes: measured,
|
|
257
|
+
total: measured.reduce(addBounds, emptyBounds()),
|
|
258
|
+
};
|
|
259
|
+
}
|
|
260
|
+
function measureNode(node) {
|
|
261
|
+
return Object.freeze({
|
|
262
|
+
messageCodeUnits: JSON.stringify(node.messages).length,
|
|
263
|
+
transcriptCodeUnits: JSON.stringify(node.blocks).length,
|
|
264
|
+
contextCodeUnits: node.context?.summary.length ?? 0,
|
|
265
|
+
});
|
|
266
|
+
}
|
|
267
|
+
function emptyBounds() {
|
|
268
|
+
return Object.freeze({ messageCodeUnits: 0, transcriptCodeUnits: 0, contextCodeUnits: 0 });
|
|
269
|
+
}
|
|
270
|
+
function addBounds(left, right) {
|
|
271
|
+
return Object.freeze({
|
|
272
|
+
messageCodeUnits: left.messageCodeUnits + right.messageCodeUnits,
|
|
273
|
+
transcriptCodeUnits: left.transcriptCodeUnits + right.transcriptCodeUnits,
|
|
274
|
+
contextCodeUnits: left.contextCodeUnits + right.contextCodeUnits,
|
|
275
|
+
});
|
|
276
|
+
}
|
|
277
|
+
function replaceBounds(total, before, after) {
|
|
278
|
+
return Object.freeze({
|
|
279
|
+
messageCodeUnits: total.messageCodeUnits - before.messageCodeUnits + after.messageCodeUnits,
|
|
280
|
+
transcriptCodeUnits: total.transcriptCodeUnits - before.transcriptCodeUnits + after.transcriptCodeUnits,
|
|
281
|
+
contextCodeUnits: total.contextCodeUnits - before.contextCodeUnits + after.contextCodeUnits,
|
|
282
|
+
});
|
|
283
|
+
}
|
|
284
|
+
function assertContextPath(nodes, owner) {
|
|
285
|
+
const context = owner.context;
|
|
286
|
+
if (context === undefined)
|
|
287
|
+
return;
|
|
288
|
+
const boundary = context.throughNodeId === owner.id
|
|
289
|
+
? owner
|
|
290
|
+
: nodes[context.throughNodeId - 1];
|
|
291
|
+
if (boundary === undefined || !validContextAnchor(context, boundary.messages.length)) {
|
|
292
|
+
throw new Error("turn context checkpoint is invalid");
|
|
293
|
+
}
|
|
294
|
+
let current = owner;
|
|
295
|
+
while (current !== undefined && current.id !== boundary.id) {
|
|
296
|
+
current = nodes[current.parentId - 1];
|
|
297
|
+
}
|
|
298
|
+
if (current === undefined)
|
|
299
|
+
throw new Error("turn context checkpoint is outside its branch");
|
|
250
300
|
}
|
|
251
301
|
function assertContextPaths(nodes, owners) {
|
|
252
302
|
if (owners.length === 0)
|
|
@@ -288,3 +338,14 @@ function validNodeId(value) {
|
|
|
288
338
|
function clone(value) {
|
|
289
339
|
return structuredClone(value);
|
|
290
340
|
}
|
|
341
|
+
function deepFreeze(value, seen = new WeakSet()) {
|
|
342
|
+
if (typeof value !== "object" || value === null || seen.has(value))
|
|
343
|
+
return value;
|
|
344
|
+
seen.add(value);
|
|
345
|
+
if (!ArrayBuffer.isView(value)) {
|
|
346
|
+
for (const child of Object.values(value))
|
|
347
|
+
deepFreeze(child, seen);
|
|
348
|
+
Object.freeze(value);
|
|
349
|
+
}
|
|
350
|
+
return value;
|
|
351
|
+
}
|
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
// The shell needs a useful process environment, not the application's secrets.
|
|
2
2
|
import { credentialValues } from "./credentials.js";
|
|
3
3
|
import { accountValues } from "./accounts.js";
|
|
4
|
+
import { USER_STORE_LIMITS } from "./user-store.js";
|
|
4
5
|
const REDACTED = "[credential redacted]";
|
|
5
6
|
const MIN_HEURISTIC_SECRET_CHARS = 8;
|
|
7
|
+
export const MAX_REDACTION_SECRETS = USER_STORE_LIMITS.credentialEntries;
|
|
8
|
+
const MAX_REDACTION_SECRET_CODE_UNITS = USER_STORE_LIMITS.accountToken;
|
|
6
9
|
const EXPLICIT_CREDENTIAL_ENVIRONMENT_NAMES = new Set([
|
|
7
10
|
"ANTHROPIC_API_KEY",
|
|
8
11
|
"OLLAMA_API_KEY",
|
|
@@ -27,11 +30,24 @@ export function shellEnvironment(source = process.env) {
|
|
|
27
30
|
}
|
|
28
31
|
/** Remove values Jecode recognizes as credentials before tool output leaves the shell boundary. */
|
|
29
32
|
export function redactCredentials(text, source = process.env) {
|
|
30
|
-
|
|
33
|
+
const known = secrets(source);
|
|
34
|
+
return known.saturated ? (text === "" ? "" : REDACTED) : redact(text, known.values);
|
|
31
35
|
}
|
|
32
36
|
/** Redact before bounded capture, retaining enough raw overlap for split values. */
|
|
33
37
|
export function credentialRedactor(source = process.env) {
|
|
34
|
-
const
|
|
38
|
+
const known = secrets(source);
|
|
39
|
+
if (known.saturated)
|
|
40
|
+
return closedRedactor();
|
|
41
|
+
const values = known.values;
|
|
42
|
+
const candidates = new Map();
|
|
43
|
+
for (const value of values) {
|
|
44
|
+
const first = value[0];
|
|
45
|
+
const bucket = candidates.get(first);
|
|
46
|
+
if (bucket === undefined)
|
|
47
|
+
candidates.set(first, [value]);
|
|
48
|
+
else
|
|
49
|
+
bucket.push(value);
|
|
50
|
+
}
|
|
35
51
|
const longest = Math.max(0, ...values.map((value) => value.length));
|
|
36
52
|
let pending = "";
|
|
37
53
|
return {
|
|
@@ -40,16 +56,17 @@ export function credentialRedactor(source = process.env) {
|
|
|
40
56
|
const ready = [];
|
|
41
57
|
let at = 0;
|
|
42
58
|
while (at < combined.length) {
|
|
43
|
-
const
|
|
59
|
+
const matching = candidates.get(combined[at]) ?? [];
|
|
60
|
+
const rest = matching.length === 0 ? "" : combined.slice(at);
|
|
44
61
|
// A complete shorter credential can also be the prefix of a longer
|
|
45
62
|
// one. Hold that ambiguous suffix until the next chunk proves which
|
|
46
63
|
// value arrived, otherwise the longer credential leaks its tail.
|
|
47
64
|
if (rest.length < longest &&
|
|
48
|
-
|
|
65
|
+
matching.some((value) => value.length > rest.length && value.startsWith(rest))) {
|
|
49
66
|
pending = rest;
|
|
50
67
|
return ready.join("");
|
|
51
68
|
}
|
|
52
|
-
const complete =
|
|
69
|
+
const complete = matching.find((value) => combined.startsWith(value, at));
|
|
53
70
|
if (complete !== undefined) {
|
|
54
71
|
ready.push(REDACTED);
|
|
55
72
|
at += complete.length;
|
|
@@ -68,6 +85,20 @@ export function credentialRedactor(source = process.env) {
|
|
|
68
85
|
},
|
|
69
86
|
};
|
|
70
87
|
}
|
|
88
|
+
function closedRedactor() {
|
|
89
|
+
let emitted = false;
|
|
90
|
+
return {
|
|
91
|
+
write(chunk) {
|
|
92
|
+
if (chunk === "" || emitted)
|
|
93
|
+
return "";
|
|
94
|
+
emitted = true;
|
|
95
|
+
return REDACTED;
|
|
96
|
+
},
|
|
97
|
+
end() {
|
|
98
|
+
return "";
|
|
99
|
+
},
|
|
100
|
+
};
|
|
101
|
+
}
|
|
71
102
|
function sensitiveEnvironmentName(name) {
|
|
72
103
|
const normalized = name
|
|
73
104
|
.replace(/([a-z0-9])([A-Z])/g, "$1_$2")
|
|
@@ -79,16 +110,32 @@ function sensitiveEnvironmentName(name) {
|
|
|
79
110
|
COMPACT_SENSITIVE_ENVIRONMENT_NAME.test(normalized));
|
|
80
111
|
}
|
|
81
112
|
function secrets(source) {
|
|
82
|
-
const values = new Set(
|
|
113
|
+
const values = new Set();
|
|
114
|
+
let saturated = false;
|
|
115
|
+
const add = (value) => {
|
|
116
|
+
if (value === "" || values.has(value) || saturated)
|
|
117
|
+
return;
|
|
118
|
+
if (value.length > MAX_REDACTION_SECRET_CODE_UNITS ||
|
|
119
|
+
values.size >= MAX_REDACTION_SECRETS) {
|
|
120
|
+
saturated = true;
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
values.add(value);
|
|
124
|
+
};
|
|
125
|
+
for (const value of [...credentialValues(), ...accountValues()])
|
|
126
|
+
add(value);
|
|
83
127
|
for (const [name, value] of Object.entries(source)) {
|
|
84
|
-
if (value === undefined || value === "")
|
|
128
|
+
if (saturated || value === undefined || value === "")
|
|
85
129
|
continue;
|
|
86
130
|
const explicit = EXPLICIT_CREDENTIAL_ENVIRONMENT_NAMES.has(name.toUpperCase());
|
|
87
131
|
if (explicit || (value.length >= MIN_HEURISTIC_SECRET_CHARS && sensitiveEnvironment(name, value))) {
|
|
88
|
-
|
|
132
|
+
add(value);
|
|
89
133
|
}
|
|
90
134
|
}
|
|
91
|
-
return
|
|
135
|
+
return {
|
|
136
|
+
values: [...values].sort((left, right) => right.length - left.length),
|
|
137
|
+
saturated,
|
|
138
|
+
};
|
|
92
139
|
}
|
|
93
140
|
function redact(text, values) {
|
|
94
141
|
let redacted = text;
|
package/dist/credentials.js
CHANGED
|
@@ -10,11 +10,11 @@
|
|
|
10
10
|
// secret in the working tree is one `git add -A` from being published, which
|
|
11
11
|
// is why "not in the repo" is a rule and not a preference.
|
|
12
12
|
import { chmod, mkdir } from "node:fs/promises";
|
|
13
|
-
import { readFileSync } from "node:fs";
|
|
14
13
|
import * as path from "node:path";
|
|
15
14
|
import { atomicWrite } from "./atomic.js";
|
|
16
15
|
import { withStoreLock } from "./store-lock.js";
|
|
17
16
|
import { legacyUserDataPath, userDataLabel, userDataPath } from "./user-data.js";
|
|
17
|
+
import { assertStoreText, readBoundedJsonSync, USER_STORE_LIMITS } from "./user-store.js";
|
|
18
18
|
/** Keys this session was given but not asked to keep. Dies with the window. */
|
|
19
19
|
const held = new Map();
|
|
20
20
|
/** The saved file, read once. `undefined` until the first look at it. */
|
|
@@ -58,6 +58,10 @@ export function hasSaved(name) {
|
|
|
58
58
|
}
|
|
59
59
|
/** Take a key for this session only. Nothing is written anywhere. */
|
|
60
60
|
export function hold(name, value) {
|
|
61
|
+
assertCredential(name, value);
|
|
62
|
+
if (!held.has(name) && held.size >= USER_STORE_LIMITS.credentialEntries) {
|
|
63
|
+
throw new Error("too many session credentials");
|
|
64
|
+
}
|
|
61
65
|
held.set(name, value);
|
|
62
66
|
}
|
|
63
67
|
/** Remove only the value held by this process. Saved and environment values remain. */
|
|
@@ -73,10 +77,14 @@ export function forgetSession(name) {
|
|
|
73
77
|
* Windows ignores the mode and relies on the profile directory's own ACL.
|
|
74
78
|
*/
|
|
75
79
|
export async function keep(name, value) {
|
|
80
|
+
assertCredential(name, value);
|
|
76
81
|
const file = storePath();
|
|
77
82
|
await prepare(file);
|
|
78
83
|
return withStoreLock(file, async () => {
|
|
79
84
|
const all = { ...readSavedStore(), [name]: value };
|
|
85
|
+
if (Object.keys(all).length > USER_STORE_LIMITS.credentialEntries) {
|
|
86
|
+
throw new Error("too many saved credentials");
|
|
87
|
+
}
|
|
80
88
|
await persist(file, all);
|
|
81
89
|
saved = all;
|
|
82
90
|
// A newly saved replacement must become active immediately. Otherwise an
|
|
@@ -131,9 +139,14 @@ function readSavedStore() {
|
|
|
131
139
|
}
|
|
132
140
|
function readStore(file) {
|
|
133
141
|
try {
|
|
134
|
-
const parsed =
|
|
142
|
+
const parsed = readBoundedJsonSync(file, USER_STORE_LIMITS.credentialsBytes);
|
|
143
|
+
if (!record(parsed))
|
|
144
|
+
return {};
|
|
135
145
|
// Anything that is not a string is not a key, whatever the file says.
|
|
136
|
-
|
|
146
|
+
const entries = Object.entries(parsed);
|
|
147
|
+
if (entries.length > USER_STORE_LIMITS.credentialEntries)
|
|
148
|
+
return {};
|
|
149
|
+
return Object.fromEntries(entries.filter((entry) => credential(entry[0], entry[1])));
|
|
137
150
|
}
|
|
138
151
|
catch (error) {
|
|
139
152
|
// Only a missing canonical file falls through to the legacy location. A
|
|
@@ -148,7 +161,22 @@ async function prepare(file) {
|
|
|
148
161
|
await chmod(directory, 0o700);
|
|
149
162
|
}
|
|
150
163
|
async function persist(file, values) {
|
|
151
|
-
|
|
164
|
+
const text = `${JSON.stringify(values, null, 2)}\n`;
|
|
165
|
+
assertStoreText(text, USER_STORE_LIMITS.credentialsBytes);
|
|
166
|
+
await atomicWrite(file, text, { mode: 0o600 });
|
|
167
|
+
}
|
|
168
|
+
function assertCredential(name, value) {
|
|
169
|
+
if (!credential(name, value))
|
|
170
|
+
throw new Error("invalid credential name or value");
|
|
171
|
+
}
|
|
172
|
+
function credential(name, value) {
|
|
173
|
+
return name.length > 0 && name.length <= USER_STORE_LIMITS.credentialName &&
|
|
174
|
+
/^[A-Za-z_][A-Za-z0-9_]*$/.test(name) &&
|
|
175
|
+
typeof value === "string" && value.length > 0 &&
|
|
176
|
+
value.length <= USER_STORE_LIMITS.credentialValue;
|
|
177
|
+
}
|
|
178
|
+
function record(value) {
|
|
179
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
152
180
|
}
|
|
153
181
|
/** An empty variable is an unset variable — an exported "" is not a key. */
|
|
154
182
|
function use(value) {
|