incremnt 0.7.2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +57 -1
- package/package.json +2 -1
- package/src/ask-answer-verifier.js +857 -0
- package/src/ask-coach.js +2634 -0
- package/src/ask-replay.js +358 -0
- package/src/auth.js +169 -15
- package/src/contract.js +160 -3
- package/src/format.js +24 -1
- package/src/lib.js +205 -17
- package/src/mcp.js +88 -24
- package/src/openrouter.js +242 -19
- package/src/plan-changeset.js +132 -0
- package/src/program-draft.js +230 -0
- package/src/prompt-changelog.js +90 -0
- package/src/promptfoo-evals.js +10 -4
- package/src/promptfoo-langfuse-scores.js +55 -0
- package/src/queries.js +992 -987
- package/src/remote.js +465 -12
- package/src/score-context.js +14 -7
- package/src/score-prelude.js +113 -0
- package/src/service-url.js +9 -0
- package/src/summary-evals.js +677 -42
- package/src/sync-service.js +1259 -352
- package/src/transport.js +119 -3
package/src/transport.js
CHANGED
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
import { readSnapshot, resolveSnapshotSource } from './local.js';
|
|
2
2
|
import { createRemoteTransport } from './remote.js';
|
|
3
|
-
import {
|
|
3
|
+
import { askMissingObservationFollowUpContext, askObservationFollowUpContext, askRoutedContext } from './ask-coach.js';
|
|
4
|
+
import { sanitizeHistory } from './prompt-security.js';
|
|
5
|
+
import { executeCoachReadTool, executeReadCommand, shouldKeepCurrentCoachObservation } from './queries.js';
|
|
4
6
|
import { isSessionExpired } from './state.js';
|
|
7
|
+
import { fetchRemoteContract, refreshRemoteSession } from './auth.js';
|
|
8
|
+
import { resolveConfiguredBaseUrl } from './service-url.js';
|
|
5
9
|
|
|
6
10
|
function prefersLocal(options) {
|
|
7
11
|
return Boolean(options.input || process.env.INCREMNT_SNAPSHOT || process.env.ONEMORE_SNAPSHOT);
|
|
@@ -13,6 +17,19 @@ function snapshotNotFoundError() {
|
|
|
13
17
|
return error;
|
|
14
18
|
}
|
|
15
19
|
|
|
20
|
+
function assertValidAskPlanQuestion(question) {
|
|
21
|
+
if (!question || typeof question !== 'string' || question.trim().length === 0) {
|
|
22
|
+
const error = new Error('question is required');
|
|
23
|
+
error.code = 'VALIDATION_ERROR';
|
|
24
|
+
throw error;
|
|
25
|
+
}
|
|
26
|
+
if (question.length > 500) {
|
|
27
|
+
const error = new Error('question must be 500 characters or fewer');
|
|
28
|
+
error.code = 'VALIDATION_ERROR';
|
|
29
|
+
throw error;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
16
33
|
function createLocalTransport(snapshotSource) {
|
|
17
34
|
return {
|
|
18
35
|
kind: 'local',
|
|
@@ -41,6 +58,44 @@ function createLocalTransport(snapshotSource) {
|
|
|
41
58
|
const snapshot = await readSnapshot(snapshotSource.path);
|
|
42
59
|
return executeCoachReadTool(snapshot, toolName, input);
|
|
43
60
|
},
|
|
61
|
+
async planAskInteraction(input = {}) {
|
|
62
|
+
if (!snapshotSource) {
|
|
63
|
+
throw snapshotNotFoundError();
|
|
64
|
+
}
|
|
65
|
+
assertValidAskPlanQuestion(input.question);
|
|
66
|
+
|
|
67
|
+
const snapshot = await readSnapshot(snapshotSource.path);
|
|
68
|
+
const exclude = new Set(String(input.exclude ?? '').split(',').map((item) => item.trim()).filter(Boolean));
|
|
69
|
+
if (input.coachObservation) {
|
|
70
|
+
const observationId = String(input.coachObservation.id ?? '').trim();
|
|
71
|
+
const snapshotObservation = (snapshot.coachObservations ?? [])
|
|
72
|
+
.find((observation) => (
|
|
73
|
+
String(observation?.id ?? '') === observationId &&
|
|
74
|
+
shouldKeepCurrentCoachObservation(observation, { includeOutcomeHistory: true })
|
|
75
|
+
));
|
|
76
|
+
const routedContext = snapshotObservation
|
|
77
|
+
? askObservationFollowUpContext(snapshot, input.question, snapshotObservation, {
|
|
78
|
+
exclude,
|
|
79
|
+
intent: input.coachObservation.intent
|
|
80
|
+
})
|
|
81
|
+
: askMissingObservationFollowUpContext(snapshot, input.question, input.coachObservation, {
|
|
82
|
+
exclude,
|
|
83
|
+
intent: input.coachObservation.intent
|
|
84
|
+
});
|
|
85
|
+
return {
|
|
86
|
+
contextBundle: routedContext.contextBundle,
|
|
87
|
+
metadata: routedContext.metadata
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
const routedContext = askRoutedContext(snapshot, input.question, {
|
|
91
|
+
exclude,
|
|
92
|
+
history: sanitizeHistory(input.history)
|
|
93
|
+
});
|
|
94
|
+
return {
|
|
95
|
+
contextBundle: routedContext.contextBundle,
|
|
96
|
+
metadata: routedContext.metadata
|
|
97
|
+
};
|
|
98
|
+
},
|
|
44
99
|
async executeWriteCommand() {
|
|
45
100
|
const error = new Error('Write commands require a remote session. Run incremnt login first.');
|
|
46
101
|
error.code = 'WRITE_REQUIRES_REMOTE';
|
|
@@ -55,10 +110,71 @@ export async function createTransport(options, sessionState) {
|
|
|
55
110
|
return createLocalTransport(snapshotSource);
|
|
56
111
|
}
|
|
57
112
|
|
|
113
|
+
const envAgentToken = String(process.env.INCREMNT_AGENT_TOKEN ?? '').trim();
|
|
114
|
+
if (envAgentToken) {
|
|
115
|
+
const baseUrl = resolveConfiguredBaseUrl(options, sessionState.session);
|
|
116
|
+
if (baseUrl) {
|
|
117
|
+
return {
|
|
118
|
+
...createRemoteTransport({
|
|
119
|
+
exists: false,
|
|
120
|
+
path: sessionState.path,
|
|
121
|
+
session: {
|
|
122
|
+
version: 1,
|
|
123
|
+
mode: 'remote',
|
|
124
|
+
account: null,
|
|
125
|
+
auth: {
|
|
126
|
+
accessToken: envAgentToken,
|
|
127
|
+
refreshToken: null,
|
|
128
|
+
expiresAt: null,
|
|
129
|
+
type: 'agent-token',
|
|
130
|
+
source: 'env',
|
|
131
|
+
access: null
|
|
132
|
+
},
|
|
133
|
+
transport: {
|
|
134
|
+
baseUrl,
|
|
135
|
+
contractVersion: null,
|
|
136
|
+
capabilities: null,
|
|
137
|
+
uncheckedContract: true
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}, {
|
|
141
|
+
credentialSource: 'env',
|
|
142
|
+
resolveContract: () => fetchRemoteContract(baseUrl, envAgentToken)
|
|
143
|
+
}),
|
|
144
|
+
expired: false,
|
|
145
|
+
snapshotSource: null
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
58
150
|
if (sessionState.session) {
|
|
59
|
-
|
|
151
|
+
let session = sessionState.session;
|
|
152
|
+
let expired = isSessionExpired(session);
|
|
153
|
+
const isAgentToken = session.auth?.type === 'agent-token';
|
|
154
|
+
const baseUrl = session.transport?.baseUrl;
|
|
155
|
+
|
|
156
|
+
if (expired && !isAgentToken && baseUrl && session.auth?.accessToken) {
|
|
157
|
+
try {
|
|
158
|
+
const refreshed = await refreshRemoteSession(baseUrl, session.auth.accessToken, session);
|
|
159
|
+
session = refreshed.session;
|
|
160
|
+
expired = false;
|
|
161
|
+
} catch (error) {
|
|
162
|
+
if (error?.code === 'SESSION_EXPIRED') {
|
|
163
|
+
expired = true;
|
|
164
|
+
} else {
|
|
165
|
+
throw error;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
60
170
|
return {
|
|
61
|
-
...createRemoteTransport(
|
|
171
|
+
...createRemoteTransport({
|
|
172
|
+
...sessionState,
|
|
173
|
+
session
|
|
174
|
+
}, {
|
|
175
|
+
expired,
|
|
176
|
+
credentialSource: session.auth?.source ?? 'session'
|
|
177
|
+
}),
|
|
62
178
|
expired,
|
|
63
179
|
snapshotSource: null
|
|
64
180
|
};
|