@jrpool/kilotest 52.0.0 → 57.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.devin/config.local.json +4 -1
- package/.github/workflows/ci.yml +3 -0
- package/README.md +1 -1
- package/api/awaitTest.test.ts +114 -0
- package/api/awaitTest.ts +108 -0
- package/api/listReports.ts +2 -2
- package/api/orderNewTest.test.ts +213 -0
- package/api/orderNewTest.ts +145 -0
- package/api/orderRetest.test.ts +179 -0
- package/api/orderRetest.ts +119 -0
- package/api/requestFeature.test.ts +36 -0
- package/api/requestFeature.ts +26 -4
- package/api/{requestTest.test.ts → requestNewTest.test.ts} +57 -23
- package/api/{requestTest.ts → requestNewTest.ts} +31 -36
- package/api/requestRetest.test.ts +24 -6
- package/api/requestRetest.ts +10 -29
- package/api/routes.test.ts +7 -4
- package/api/routes.ts +45 -7
- package/api/schemas.ts +120 -10
- package/api/util.ts +59 -2
- package/api/version.ts +6 -2
- package/capability.md +9 -6
- package/docs/AI-TOOLS.md +3 -3
- package/docs/SERVICE.md +19 -8
- package/env.example +39 -0
- package/index.test.ts +146 -24
- package/index.ts +129 -36
- package/lhm.plugin.json +1 -1
- package/linkValidation.test.ts +2 -2
- package/llms-full.txt +35 -4
- package/llms.txt +6 -3
- package/mcp.test.ts +109 -12
- package/mcp.ts +146 -26
- package/memory/MEMORY.md +1 -0
- package/memory/plan_issue_111_instant_test.md +198 -0
- package/openapi.yaml +456 -9
- package/package.json +2 -1
- package/routes.test.ts +1 -1
- package/routing.test.ts +2 -2
- package/scripts/generate-openapi.ts +1 -1
- package/server.json +1 -1
- package/sitemap.xml +1 -1
- package/smokeTest.ts +2 -2
- package/util.test.ts +486 -2
- package/util.ts +276 -45
- package/web/listReports/index.html +1 -1
- package/web/listReports/listReports.test.ts +1 -1
- package/web/{requestTest → requestNewTest}/index.ts +10 -1
- package/web/{requestTest/requestTest.test.ts → requestNewTest/requestNewTest.test.ts} +18 -2
- package/web/{requestTestForm → requestNewTestForm}/index.html +1 -1
- package/web/{requestTestForm/requestTestForm.test.ts → requestNewTestForm/requestNewTestForm.test.ts} +6 -6
- package/web/requestRetest/index.ts +9 -0
- package/web/requestRetest/requestRetest.test.ts +14 -0
- package/web/tutorialAI/index.ts +25 -3
- package/web/tutorialAI/tutorialAI.test.ts +32 -0
- package/web/tutorialWeb/index.ts +25 -3
- package/web/tutorialWeb/tutorialWeb.test.ts +32 -0
- /package/web/{requestTest → requestNewTest}/index.html +0 -0
- /package/web/{requestTestForm → requestNewTestForm}/index.ts +0 -0
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
/*
|
|
2
|
+
orderRetest.test.ts
|
|
3
|
+
Tests for api/orderRetest.ts using the fixture corpus.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
// IMPORTS
|
|
7
|
+
|
|
8
|
+
import {test, before, beforeEach, after} from 'node:test';
|
|
9
|
+
import assert from 'node:assert/strict';
|
|
10
|
+
import fs from 'node:fs/promises';
|
|
11
|
+
import path from 'node:path';
|
|
12
|
+
import {fixtureDBDir} from '../test/dbFixture.ts';
|
|
13
|
+
|
|
14
|
+
for (const key of ['MANAGER_EMAIL', 'ALERT_API_HOST', 'ALERT_API_PATH', 'ALERT_API_KEY', 'ALERT_FROM']) {
|
|
15
|
+
process.env[key] = '';
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
// SETUP AND TEARDOWN
|
|
19
|
+
|
|
20
|
+
const savedDBDir = process.env.DB_DIR;
|
|
21
|
+
const queuePath = path.join(fixtureDBDir, 'jobs', 'queue');
|
|
22
|
+
// A report added only to this test's disposable copy of the fixture database (never to
|
|
23
|
+
// the tracked test/fixtures/db corpus), matching the fixture requestRetest.test.ts uses
|
|
24
|
+
// for the same reason: a 3-character job ID satisfying isJobID without perturbing
|
|
25
|
+
// report-count tests elsewhere that copy the whole corpus.
|
|
26
|
+
const retestFixturePath = path.join(fixtureDBDir, 'reports', '260101T0004-ord.json');
|
|
27
|
+
const retestFixtureReport = {
|
|
28
|
+
id: '260101T0004-ord',
|
|
29
|
+
what: 'Order Retest Fixture Page',
|
|
30
|
+
target: {what: 'Order Retest Fixture Page', url: 'https://example.com/orderretestfixture'},
|
|
31
|
+
sources: {worker: 'test-worker'},
|
|
32
|
+
acts: [],
|
|
33
|
+
jobData: {startTime: '26-01-01T00:00', endTime: '26-01-01T00:10', elapsedSeconds: 600, preventions: {}, issuelessRules: []},
|
|
34
|
+
catalog: {},
|
|
35
|
+
images: {},
|
|
36
|
+
checkpoints: []
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
before(async () => {
|
|
40
|
+
process.env.DB_DIR = fixtureDBDir;
|
|
41
|
+
await fs.writeFile(retestFixturePath, JSON.stringify(retestFixtureReport));
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
beforeEach(async () => {
|
|
45
|
+
for (const fileName of await fs.readdir(queuePath)) {
|
|
46
|
+
await fs.unlink(path.join(queuePath, fileName));
|
|
47
|
+
}
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
import {response} from './orderRetest.ts';
|
|
51
|
+
|
|
52
|
+
after(async () => {
|
|
53
|
+
await fs.unlink(retestFixturePath);
|
|
54
|
+
if (savedDBDir !== undefined) {
|
|
55
|
+
process.env.DB_DIR = savedDBDir;
|
|
56
|
+
}
|
|
57
|
+
else {
|
|
58
|
+
delete process.env.DB_DIR;
|
|
59
|
+
}
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
// TESTS
|
|
63
|
+
|
|
64
|
+
test('orderRetest rejects a nonexistent report', async () => {
|
|
65
|
+
const body = await response(['251231T0000', 'zzz', 'A reason that is long enough.']);
|
|
66
|
+
const details = body['response content']['details about your order'] as any;
|
|
67
|
+
assert.ok(details.error.includes('does not exist'));
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
test('orderRetest rejects a malformed timestamp or job identifier', async () => {
|
|
71
|
+
const body = await response(['999999T9999', 'xyz', 'A reason that is long enough.']);
|
|
72
|
+
const details = body['response content']['details about your order'] as any;
|
|
73
|
+
assert.ok(details.error.includes('malformed'));
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
test('orderRetest rejects a reason shorter than 20 characters', async () => {
|
|
77
|
+
const body = await response(['260101T0004', 'ord', 'short']);
|
|
78
|
+
const details = body['response content']['details about your order'] as any;
|
|
79
|
+
assert.ok(details.error.includes('reason'));
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
test('orderRetest rejects a reason longer than 100 characters', async () => {
|
|
83
|
+
const longReason = 'x'.repeat(101);
|
|
84
|
+
const body = await response(['260101T0004', 'ord', longReason]);
|
|
85
|
+
const details = body['response content']['details about your order'] as any;
|
|
86
|
+
assert.ok(details.error.includes('reason'));
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
test('orderRetest rejects a superseded report', async () => {
|
|
90
|
+
// 260101T0000-mix is an earlier report of "Mixed Outcomes Page" than 260202T0000-new,
|
|
91
|
+
// so it is superseded.
|
|
92
|
+
const body = await response(['260101T0000', 'mix', 'A reason that is long enough.']);
|
|
93
|
+
const details = body['response content']['details about your order'] as any;
|
|
94
|
+
assert.equal(details.error, undefined);
|
|
95
|
+
const disposition = body['response content']['disposition of your order'] as any;
|
|
96
|
+
assert.ok(
|
|
97
|
+
disposition['what happens next'].includes('a later report about a page with the same description is available.')
|
|
98
|
+
);
|
|
99
|
+
assert.equal(disposition['report identifier'], null);
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
test('orderRetest rejects a report matching a claimed job by description', async () => {
|
|
103
|
+
const claimedPath = path.join(fixtureDBDir, 'jobs', 'claimed', 'clm.json');
|
|
104
|
+
await fs.writeFile(claimedPath, JSON.stringify({
|
|
105
|
+
target: {what: 'Order Retest Fixture Page', url: 'https://example.com/unrelated'}
|
|
106
|
+
}));
|
|
107
|
+
try {
|
|
108
|
+
const body = await response(['260101T0004', 'ord', 'A reason that is long enough.']);
|
|
109
|
+
const details = body['response content']['details about your order'] as any;
|
|
110
|
+
assert.equal(details.error, undefined);
|
|
111
|
+
const disposition = body['response content']['disposition of your order'] as any;
|
|
112
|
+
assert.ok(
|
|
113
|
+
disposition['what happens next']
|
|
114
|
+
.includes('a request to test a page with the same description is already approved.')
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
finally {
|
|
118
|
+
await fs.unlink(claimedPath);
|
|
119
|
+
}
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
test('orderRetest rejects a report matching a queued job by URL', async () => {
|
|
123
|
+
const queuedFilePath = path.join(queuePath, 'que.json');
|
|
124
|
+
await fs.writeFile(queuedFilePath, JSON.stringify({
|
|
125
|
+
target: {what: 'Some Other Page', url: 'https://example.com/orderretestfixture'}
|
|
126
|
+
}));
|
|
127
|
+
try {
|
|
128
|
+
const body = await response(['260101T0004', 'ord', 'A reason that is long enough.']);
|
|
129
|
+
const details = body['response content']['details about your order'] as any;
|
|
130
|
+
assert.equal(details.error, undefined);
|
|
131
|
+
const disposition = body['response content']['disposition of your order'] as any;
|
|
132
|
+
assert.ok(
|
|
133
|
+
disposition['what happens next']
|
|
134
|
+
.includes('a request to test a page with the same URL is already approved.')
|
|
135
|
+
);
|
|
136
|
+
}
|
|
137
|
+
finally {
|
|
138
|
+
await fs.unlink(queuedFilePath);
|
|
139
|
+
}
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
test('orderRetest rejects an order once the job queue is full', async () => {
|
|
143
|
+
const fillerPath = path.join(queuePath, 'fil.json');
|
|
144
|
+
await fs.writeFile(fillerPath, JSON.stringify({target: {what: 'Filler', url: 'https://example.com/filler'}}));
|
|
145
|
+
process.env.JOB_QUEUE_MAX = '1';
|
|
146
|
+
try {
|
|
147
|
+
const body = await response(['260101T0004', 'ord', 'A reason that is long enough.']);
|
|
148
|
+
const details = body['response content']['details about your order'] as any;
|
|
149
|
+
assert.equal(details.error, undefined);
|
|
150
|
+
const disposition = body['response content']['disposition of your order'] as any;
|
|
151
|
+
assert.ok(disposition['what happens next'].includes('too many requests are awaiting approval right now.'));
|
|
152
|
+
} finally {
|
|
153
|
+
delete process.env.JOB_QUEUE_MAX;
|
|
154
|
+
await fs.unlink(fillerPath);
|
|
155
|
+
}
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
test('orderRetest accepts a valid retest order and enqueues a job directly', async () => {
|
|
159
|
+
const body = await response(['260101T0004', 'ord', 'A reason that is long enough.']);
|
|
160
|
+
const details = body['response content']['details about your order'] as any;
|
|
161
|
+
assert.equal(details.error, undefined);
|
|
162
|
+
assert.equal(details['page to be retested'].description, 'Order Retest Fixture Page');
|
|
163
|
+
const disposition = body['response content']['disposition of your order'] as any;
|
|
164
|
+
assert.ok(disposition['report identifier']);
|
|
165
|
+
assert.ok(disposition['what happens next'].includes('Testing has begun.'));
|
|
166
|
+
assert.ok(disposition['do you want to wait for completion'].includes('awaitTest'));
|
|
167
|
+
const queuedFiles = await fs.readdir(queuePath);
|
|
168
|
+
assert.equal(queuedFiles.length, 1);
|
|
169
|
+
const job = JSON.parse(await fs.readFile(path.join(queuePath, queuedFiles[0]!), 'utf8'));
|
|
170
|
+
assert.equal(job.target.what, 'Order Retest Fixture Page');
|
|
171
|
+
assert.equal(job.sources.reason, 'A reason that is long enough.');
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
test('orderRetest response omits a web UI URL for this request but includes the ancestor', async () => {
|
|
175
|
+
const body = await response(['260101T0004', 'ord', 'A reason that is long enough.']);
|
|
176
|
+
const similarWeb = body['URLs of similar requests for web users'] as any;
|
|
177
|
+
assert.equal(similarWeb['this request'], null);
|
|
178
|
+
assert.ok(similarWeb['closest ancestor request'].includes('listIssues.html'));
|
|
179
|
+
});
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/*
|
|
2
|
+
orderRetest.ts
|
|
3
|
+
Processes an order to retest a page immediately, without manual approval, and returns an acknowledgement.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
// IMPORTS
|
|
7
|
+
|
|
8
|
+
import {z} from 'zod';
|
|
9
|
+
import {getRequestFailureReason, getResponseMetadata, getThisHost, getToolsFacts} from './util.ts';
|
|
10
|
+
import {checkLength, isJobID, isTimeStamp, orderJob} from '../util.ts';
|
|
11
|
+
import {orderRetestResponseSchema} from './schemas.ts';
|
|
12
|
+
|
|
13
|
+
// TYPES
|
|
14
|
+
|
|
15
|
+
// The response content defined by the response schema.
|
|
16
|
+
type ResponseContent = z.infer<typeof orderRetestResponseSchema>['response content'];
|
|
17
|
+
|
|
18
|
+
// FUNCTIONS
|
|
19
|
+
|
|
20
|
+
// Returns the response body.
|
|
21
|
+
export const response = async (args: string[]) => {
|
|
22
|
+
const [timeStamp = '', jobID = '', reason = ''] = args;
|
|
23
|
+
const thisHost = getThisHost();
|
|
24
|
+
const reasonCheck = checkLength(reason, 20, 100, 'reason');
|
|
25
|
+
// Initialize the response-content properties.
|
|
26
|
+
let requestDetails: ResponseContent['details about your order'];
|
|
27
|
+
let requestDisposition: ResponseContent['disposition of your order'] = null;
|
|
28
|
+
// If the timestamp or job ID is not syntactically valid:
|
|
29
|
+
if (!isTimeStamp(timeStamp) || !isJobID(jobID)) {
|
|
30
|
+
requestDetails = {
|
|
31
|
+
error: 'order invalid: the report timestamp or job identifier is malformed'
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
// Otherwise, if the encoded reason is too short or too long:
|
|
35
|
+
else if (reasonCheck.status === 'error') {
|
|
36
|
+
requestDetails = {
|
|
37
|
+
error: `order invalid: your ${reasonCheck.message}`
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
// Otherwise, i.e. if the order is facially valid:
|
|
41
|
+
else {
|
|
42
|
+
// Order the retest. Unlike processTestRequest, this enqueues the job directly, with
|
|
43
|
+
// no manual-approval step, gated only by the lightweight checks orderJob itself
|
|
44
|
+
// performs (the target is not already claimed or queued, and the job queue is not
|
|
45
|
+
// already full). This resolves the cited report's description and URL itself.
|
|
46
|
+
const orderResult = await orderJob({timeStamp, jobID}, reason);
|
|
47
|
+
// If the cited report does not exist:
|
|
48
|
+
if (orderResult.result === 'nonreport') {
|
|
49
|
+
requestDetails = {
|
|
50
|
+
error: 'order invalid: the specified report does not exist'
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
// Otherwise, i.e. if the cited report was found:
|
|
54
|
+
else {
|
|
55
|
+
// Add details about the order.
|
|
56
|
+
requestDetails = {
|
|
57
|
+
'date and time received': new Date().toISOString(),
|
|
58
|
+
'page to be retested': {
|
|
59
|
+
description: orderResult.description,
|
|
60
|
+
URL: orderResult.url
|
|
61
|
+
},
|
|
62
|
+
'reason why the page should be retested': reason
|
|
63
|
+
};
|
|
64
|
+
// Add information about the disposition of the order.
|
|
65
|
+
if (orderResult.result === 'ok') {
|
|
66
|
+
const [reportTimeStamp, reportJobID] = orderResult.jobID.split('-') as [string, string];
|
|
67
|
+
requestDisposition = {
|
|
68
|
+
'what happens next': 'Testing has begun. Once it finishes, the report will be available via the listIssues tool.',
|
|
69
|
+
'report identifier': {timeStamp: reportTimeStamp, jobID: reportJobID},
|
|
70
|
+
'how you can check for completion': `Call the listIssues tool with timeStamp "${reportTimeStamp}" and jobID "${reportJobID}" to learn whether the report is ready.`,
|
|
71
|
+
'do you want to wait for completion': 'Testing typically takes about 2 minutes and occasionally up to 4 minutes. Do you want to wait (and be notified of completion) instead of checking back later? If so, call the awaitTest tool with the same timeStamp and jobID.'
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
else {
|
|
75
|
+
requestDisposition = {
|
|
76
|
+
'what happens next': `Your order will not be processed, because ${getRequestFailureReason(
|
|
77
|
+
orderResult.result,
|
|
78
|
+
{result: 'superseded', reason: 'a later report about a page with the same description is available.'}
|
|
79
|
+
)}`,
|
|
80
|
+
'report identifier': null,
|
|
81
|
+
'how you can check for completion': 'Not applicable.',
|
|
82
|
+
'do you want to wait for completion': 'Not applicable.'
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
// Create the response content.
|
|
88
|
+
const responseContent: ResponseContent = {
|
|
89
|
+
'details about your order': requestDetails,
|
|
90
|
+
'disposition of your order': requestDisposition
|
|
91
|
+
};
|
|
92
|
+
// Create a response body.
|
|
93
|
+
const body = {
|
|
94
|
+
'tool collection': getToolsFacts(),
|
|
95
|
+
'tool name': 'orderRetest',
|
|
96
|
+
'this request': {
|
|
97
|
+
description: 'Process my order to retest a page, immediately and without manual approval. The timeStamp and jobID parameters identify the latest available report about the page. Those parameters were in the response to my earlier listIssues request. The reason property of the request body is the reason why the page should be retested.',
|
|
98
|
+
method: 'POST',
|
|
99
|
+
URL: `${thisHost}/api/orderRetest/${timeStamp}/${jobID}`,
|
|
100
|
+
body: {
|
|
101
|
+
reason
|
|
102
|
+
},
|
|
103
|
+
'closest ancestor request': {
|
|
104
|
+
'tool name': 'listIssues',
|
|
105
|
+
description: 'Provide details about one report, including basics about the issues reported in it.',
|
|
106
|
+
method: 'GET',
|
|
107
|
+
URL: `${thisHost}/api/listIssues/${timeStamp}/${jobID}`
|
|
108
|
+
}
|
|
109
|
+
},
|
|
110
|
+
'URLs of similar requests for web users': {
|
|
111
|
+
'this request': null,
|
|
112
|
+
'closest ancestor request': `${thisHost}/listIssues.html/${timeStamp}/${jobID}`
|
|
113
|
+
},
|
|
114
|
+
'response metadata': getResponseMetadata(),
|
|
115
|
+
'response content': responseContent
|
|
116
|
+
};
|
|
117
|
+
// Return it.
|
|
118
|
+
return body;
|
|
119
|
+
};
|
|
@@ -81,9 +81,45 @@ test('requestFeature accepts a non-empty feature request and notifies the manage
|
|
|
81
81
|
assert.ok(logged.some(line =>
|
|
82
82
|
line.startsWith('WARNING (Kilotest: MCP feature request received)')
|
|
83
83
|
&& line.includes('Add a dark mode toggle')
|
|
84
|
+
// The alert reports the resulting queue size, so a maintainer who has been away can
|
|
85
|
+
// see at a glance how close the queue is to full.
|
|
86
|
+
&& line.includes('Feature requests now awaiting review: 1 of 20')
|
|
84
87
|
));
|
|
85
88
|
});
|
|
86
89
|
|
|
90
|
+
test('requestFeature rejects a request with a fallback channel when the queue is full', async () => {
|
|
91
|
+
// Fill featureRequests.json with 20 distinct requests, reaching the cap.
|
|
92
|
+
const filler = Array.from({length: 20}, (_, i) => ({
|
|
93
|
+
timeStamp: '260101T0000', content: `Filler feature request number ${i}, long enough to pass the length check.`
|
|
94
|
+
}));
|
|
95
|
+
await fs.writeFile(process.env.FEATURE_REQUESTS_PATH!, JSON.stringify(filler));
|
|
96
|
+
const body = await response(['One feature request too many, long enough to pass the length check.']);
|
|
97
|
+
const details = body['response content']['details about your request'] as any;
|
|
98
|
+
assert.ok(details.error.includes('too many feature requests are awaiting review right now.'));
|
|
99
|
+
assert.ok(details.error.includes('https://github.com/jrpool/kilotest/issues'));
|
|
100
|
+
assert.ok(details.error.includes('info@kilotest.com'));
|
|
101
|
+
const featureRequests = JSON.parse(await fs.readFile(process.env.FEATURE_REQUESTS_PATH!, 'utf8'));
|
|
102
|
+
// The rejected request was not recorded.
|
|
103
|
+
assert.equal(featureRequests.length, 20);
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
test('requestFeature never rejects for a full inbox when FEATURE_REQUESTS_MAX is 0 (no limit)', async () => {
|
|
107
|
+
process.env.FEATURE_REQUESTS_MAX = '0';
|
|
108
|
+
try {
|
|
109
|
+
// Fill featureRequests.json past the default cap.
|
|
110
|
+
const filler = Array.from({length: 25}, (_, i) => ({
|
|
111
|
+
timeStamp: '260101T0000', content: `Filler feature request number ${i}, long enough to pass the length check.`
|
|
112
|
+
}));
|
|
113
|
+
await fs.writeFile(process.env.FEATURE_REQUESTS_PATH!, JSON.stringify(filler));
|
|
114
|
+
const body = await response(['One more feature request, long enough to pass the length check.']);
|
|
115
|
+
const details = body['response content']['details about your request'] as any;
|
|
116
|
+
assert.equal(details.error, undefined);
|
|
117
|
+
}
|
|
118
|
+
finally {
|
|
119
|
+
delete process.env.FEATURE_REQUESTS_MAX;
|
|
120
|
+
}
|
|
121
|
+
});
|
|
122
|
+
|
|
87
123
|
test('requestFeature includes tool name and metadata', async () => {
|
|
88
124
|
const body = await response(['A sufficiently long feature description']);
|
|
89
125
|
assert.equal(body['tool name'], 'requestFeature');
|
package/api/requestFeature.ts
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
import {z} from 'zod';
|
|
9
9
|
import {getResponseMetadata, getThisHost, getToolsFacts} from './util.ts';
|
|
10
10
|
import {sendAlert} from '../alerts.ts';
|
|
11
|
-
import {checkCommentDuplicate, checkLength, getJSON, getNowStamp} from '../util.ts';
|
|
11
|
+
import {checkCommentDuplicate, checkLength, describeMax, getEnvMax, getJSON, getNowStamp} from '../util.ts';
|
|
12
12
|
import {requestFeatureResponseSchema} from './schemas.ts';
|
|
13
13
|
import fs from 'node:fs/promises';
|
|
14
14
|
import path from 'node:path';
|
|
@@ -21,6 +21,13 @@ type ResponseContent = z.infer<typeof requestFeatureResponseSchema>['response co
|
|
|
21
21
|
// FUNCTIONS (helpers)
|
|
22
22
|
|
|
23
23
|
const getFeatureRequestsPath = () => process.env.FEATURE_REQUESTS_PATH || path.join(import.meta.dirname, '../db/featureRequests.json');
|
|
24
|
+
// Maximum number of feature requests held at once. Once reached, a new, distinct request
|
|
25
|
+
// is rejected rather than stored, so the feature-requests file and the manager alert
|
|
26
|
+
// emails it generates cannot grow without bound (modeled on a full mailbox rejecting new
|
|
27
|
+
// mail; see GitHub issue #3, abuse type 3). Configurable via FEATURE_REQUESTS_MAX, since
|
|
28
|
+
// different Kilotest deployments' maintainers may want a different size; a value of 0
|
|
29
|
+
// means no limit.
|
|
30
|
+
const getFeatureRequestsMax = () => getEnvMax('FEATURE_REQUESTS_MAX', 20);
|
|
24
31
|
|
|
25
32
|
// FUNCTIONS
|
|
26
33
|
|
|
@@ -61,7 +68,18 @@ export const response = async (args: string[]) => {
|
|
|
61
68
|
}
|
|
62
69
|
};
|
|
63
70
|
}
|
|
64
|
-
// Otherwise,
|
|
71
|
+
// Otherwise, if the feature requests already on file have reached the cap (a cap of 0
|
|
72
|
+
// means no limit, so the queue is never full):
|
|
73
|
+
else if (getFeatureRequestsMax() > 0 && featureRequests.length >= getFeatureRequestsMax()) {
|
|
74
|
+
responseContent = {
|
|
75
|
+
'details about your request': {
|
|
76
|
+
error: 'request invalid: too many feature requests are awaiting review right now. ' +
|
|
77
|
+
'Please try again later, or post your request at https://github.com/jrpool/kilotest/issues ' +
|
|
78
|
+
'or email info@kilotest.com.'
|
|
79
|
+
}
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
// Otherwise, i.e. if it is not a duplicate and the queue is not full:
|
|
65
83
|
else {
|
|
66
84
|
// Record the feature request alongside the existing ones.
|
|
67
85
|
featureRequests.push({
|
|
@@ -70,8 +88,12 @@ export const response = async (args: string[]) => {
|
|
|
70
88
|
});
|
|
71
89
|
await fs.mkdir(path.dirname(featureRequestsPath), {recursive: true});
|
|
72
90
|
await fs.writeFile(featureRequestsPath, getJSON(featureRequests));
|
|
73
|
-
// Notify the manager
|
|
74
|
-
|
|
91
|
+
// Notify the manager, including the resulting count, so a maintainer who has been
|
|
92
|
+
// away sees at a glance how urgently the requests need review.
|
|
93
|
+
await sendAlert(
|
|
94
|
+
'Kilotest: MCP feature request received',
|
|
95
|
+
`${feature}\nFeature requests now awaiting review: ${featureRequests.length} of ${describeMax(getFeatureRequestsMax())}`
|
|
96
|
+
);
|
|
75
97
|
// Add the disposition to the response content.
|
|
76
98
|
responseContent = {
|
|
77
99
|
'details about your request': {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/*
|
|
2
|
-
|
|
3
|
-
Tests for api/
|
|
2
|
+
requestNewTest.test.ts
|
|
3
|
+
Tests for api/requestNewTest.ts using the fixture corpus.
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
// IMPORTS
|
|
@@ -11,7 +11,7 @@ import fs from 'node:fs/promises';
|
|
|
11
11
|
import path from 'node:path';
|
|
12
12
|
import {fixtureDBDir} from '../test/dbFixture.ts';
|
|
13
13
|
|
|
14
|
-
// Blank the alert configuration unconditionally, before ./
|
|
14
|
+
// Blank the alert configuration unconditionally, before ./requestNewTest.ts (whose
|
|
15
15
|
// processTestRequest calls sendAlert) is ever imported below, so this file sends no
|
|
16
16
|
// real alert emails even when run directly rather than via `npm test`.
|
|
17
17
|
for (const key of ['MANAGER_EMAIL', 'ALERT_API_HOST', 'ALERT_API_PATH', 'ALERT_API_KEY', 'ALERT_FROM']) {
|
|
@@ -41,7 +41,7 @@ beforeEach(async () => {
|
|
|
41
41
|
console.log = (...args) => logged.push(args.join(' '));
|
|
42
42
|
});
|
|
43
43
|
|
|
44
|
-
import {response} from './
|
|
44
|
+
import {response} from './requestNewTest.ts';
|
|
45
45
|
|
|
46
46
|
after(() => {
|
|
47
47
|
console.log = originalLog;
|
|
@@ -55,59 +55,75 @@ after(() => {
|
|
|
55
55
|
|
|
56
56
|
// TESTS
|
|
57
57
|
|
|
58
|
-
test('
|
|
58
|
+
test('requestNewTest rejects an empty description', async () => {
|
|
59
59
|
const body = await response(['', 'https://example.com/test', 'A reason that is long enough.']);
|
|
60
60
|
assert.ok((body['response content']['details about your request'] as any).error);
|
|
61
|
-
assert.ok(!logged.some(line => line.includes('new test
|
|
61
|
+
assert.ok(!logged.some(line => line.includes('request for a new test awaits approval')));
|
|
62
62
|
});
|
|
63
63
|
|
|
64
|
-
test('
|
|
64
|
+
test('requestNewTest rejects a description longer than 100 characters', async () => {
|
|
65
65
|
const longWhat = 'x'.repeat(101);
|
|
66
66
|
const body = await response([longWhat, 'https://example.com/test', 'A reason that is long enough.']);
|
|
67
67
|
assert.ok((body['response content']['details about your request'] as any).error);
|
|
68
|
-
assert.ok(!logged.some(line => line.includes('new test
|
|
68
|
+
assert.ok(!logged.some(line => line.includes('request for a new test awaits approval')));
|
|
69
69
|
});
|
|
70
70
|
|
|
71
|
-
test('
|
|
71
|
+
test('requestNewTest rejects a URL shorter than 12 characters', async () => {
|
|
72
72
|
const body = await response(['Test Page', 'short', 'A reason that is long enough.']);
|
|
73
73
|
assert.ok((body['response content']['details about your request'] as any).error);
|
|
74
|
-
assert.ok(!logged.some(line => line.includes('new test
|
|
74
|
+
assert.ok(!logged.some(line => line.includes('request for a new test awaits approval')));
|
|
75
75
|
});
|
|
76
76
|
|
|
77
|
-
test('
|
|
77
|
+
test('requestNewTest rejects a syntactically invalid URL with the correct length', async () => {
|
|
78
78
|
const body = await response(['Test Page', 'not-a-valid-url', 'A reason that is long enough.']);
|
|
79
79
|
const details = body['response content']['details about your request'] as any;
|
|
80
80
|
assert.ok(details.error.includes('invalid URL'));
|
|
81
|
-
assert.ok(!logged.some(line => line.includes('new test
|
|
81
|
+
assert.ok(!logged.some(line => line.includes('request for a new test awaits approval')));
|
|
82
82
|
});
|
|
83
83
|
|
|
84
|
-
test('
|
|
84
|
+
test('requestNewTest rejects a private-address URL unless internal targets are allowed', async () => {
|
|
85
85
|
delete process.env.ALLOW_INTERNAL_TARGETS;
|
|
86
86
|
try {
|
|
87
87
|
const body = await response(['Internal Page', 'https://192.168.1.1/page', 'A reason that is long enough.']);
|
|
88
88
|
const details = body['response content']['details about your request'] as any;
|
|
89
89
|
assert.ok(details.error.includes('does not resolve to a page that this deployment can test'));
|
|
90
|
-
assert.ok(!logged.some(line => line.includes('new test
|
|
90
|
+
assert.ok(!logged.some(line => line.includes('request for a new test awaits approval')));
|
|
91
91
|
} finally {
|
|
92
92
|
process.env.ALLOW_INTERNAL_TARGETS = 'true';
|
|
93
93
|
}
|
|
94
94
|
});
|
|
95
95
|
|
|
96
|
-
test('
|
|
96
|
+
test('requestNewTest rejects a URL that redirects to a disallowed target, and alerts a manager', async (t) => {
|
|
97
|
+
delete process.env.ALLOW_INTERNAL_TARGETS;
|
|
98
|
+
t.mock.method(globalThis, 'fetch', async () => ({url: 'https://10.0.0.5/page'}) as Response);
|
|
99
|
+
try {
|
|
100
|
+
const body = await response(['Redirecting Page', 'https://example.com/redirector', 'A reason that is long enough.']);
|
|
101
|
+
const details = body['response content']['details about your request'] as any;
|
|
102
|
+
assert.ok(details.error.includes('does not resolve to a page that this deployment can test'));
|
|
103
|
+
assert.ok(logged.some(line =>
|
|
104
|
+
line.includes('WARNING (Kilotest: request rejected for redirecting to a disallowed target)')
|
|
105
|
+
&& line.includes('https://example.com/redirector')
|
|
106
|
+
));
|
|
107
|
+
} finally {
|
|
108
|
+
process.env.ALLOW_INTERNAL_TARGETS = 'true';
|
|
109
|
+
}
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
test('requestNewTest rejects an already-tested page', async () => {
|
|
97
113
|
const body = await response(['Mixed Outcomes Page', 'https://example.com/mixed', 'A reason that is long enough.']);
|
|
98
114
|
const details = body['response content']['details about your request'] as any;
|
|
99
115
|
// A report for this description and URL already exists, so processTestRequest
|
|
100
|
-
// returns '
|
|
116
|
+
// returns 'reportExists' and the failure is reported via the disposition, not details.error.
|
|
101
117
|
assert.equal(details.error, undefined);
|
|
102
118
|
const disposition = body['response content']['disposition of your request'] as any;
|
|
103
119
|
assert.ok(
|
|
104
120
|
disposition['what happens next']
|
|
105
121
|
.includes('a report about a page with the same description and URL is available.')
|
|
106
122
|
);
|
|
107
|
-
assert.ok(!logged.some(line => line.includes('new test
|
|
123
|
+
assert.ok(!logged.some(line => line.includes('request for a new test awaits approval')));
|
|
108
124
|
});
|
|
109
125
|
|
|
110
|
-
test('
|
|
126
|
+
test('requestNewTest rejects a page matching a claimed job by description', async () => {
|
|
111
127
|
const claimedPath = path.join(fixtureDBDir, 'jobs', 'claimed', 'clm.json');
|
|
112
128
|
await fs.writeFile(claimedPath, JSON.stringify({
|
|
113
129
|
target: {what: 'Claimed Test Page', url: 'https://example.com/claimed-job'}
|
|
@@ -127,7 +143,7 @@ test('requestTest rejects a page matching a claimed job by description', async (
|
|
|
127
143
|
}
|
|
128
144
|
});
|
|
129
145
|
|
|
130
|
-
test('
|
|
146
|
+
test('requestNewTest rejects a page matching a queued job by URL', async () => {
|
|
131
147
|
const queuedPath = path.join(fixtureDBDir, 'jobs', 'queue', 'que.json');
|
|
132
148
|
await fs.writeFile(queuedPath, JSON.stringify({
|
|
133
149
|
target: {what: 'Some Other Page', url: 'https://example.com/queued-url'}
|
|
@@ -147,7 +163,7 @@ test('requestTest rejects a page matching a queued job by URL', async () => {
|
|
|
147
163
|
}
|
|
148
164
|
});
|
|
149
165
|
|
|
150
|
-
test('
|
|
166
|
+
test('requestNewTest rejects a duplicate request', async () => {
|
|
151
167
|
await response(['Duplicate API Page', 'https://example.com/dup-api', 'A reason that is long enough.']);
|
|
152
168
|
const body = await response(['Duplicate API Page', 'https://example.com/dup-api', 'A reason that is long enough.']);
|
|
153
169
|
const details = body['response content']['details about your request'] as any;
|
|
@@ -156,21 +172,39 @@ test('requestTest rejects a duplicate request', async () => {
|
|
|
156
172
|
assert.ok(disposition['what happens next'].includes('an identical request is already awaiting approval.'));
|
|
157
173
|
});
|
|
158
174
|
|
|
159
|
-
test('
|
|
175
|
+
test('requestNewTest accepts a valid new page request', async () => {
|
|
160
176
|
const body = await response(['Brand New Page', 'https://example.com/brandnew', 'A reason that is long enough.']);
|
|
161
177
|
const details = body['response content']['details about your request'] as any;
|
|
162
178
|
assert.equal(details.error, undefined);
|
|
163
179
|
assert.equal(details['page to be tested'].description, 'Brand New Page');
|
|
164
180
|
assert.ok(logged.some(line =>
|
|
165
|
-
line.startsWith('WARNING (Kilotest: new test
|
|
181
|
+
line.startsWith('WARNING (Kilotest: request for a new test awaits approval)')
|
|
166
182
|
&& line.includes('Brand New Page')
|
|
167
183
|
&& line.includes('https://example.com/brandnew')
|
|
184
|
+
// The alert reports the resulting queue size, so a maintainer who has been away can
|
|
185
|
+
// see at a glance how close the queue is to full.
|
|
186
|
+
&& line.includes('Requests now awaiting approval: 1 of 20')
|
|
168
187
|
));
|
|
169
188
|
});
|
|
170
189
|
|
|
171
|
-
test('
|
|
190
|
+
test('requestNewTest includes disposition information for a valid request', async () => {
|
|
172
191
|
const body = await response(['Brand New Page', 'https://example.com/brandnew', 'A reason that is long enough.']);
|
|
173
192
|
const disposition = body['response content']['disposition of your request'];
|
|
174
193
|
assert.ok(disposition);
|
|
175
194
|
assert.ok(disposition['what happens next']);
|
|
176
195
|
});
|
|
196
|
+
|
|
197
|
+
test('requestNewTest rejects a request with a fallback channel when the queue is full', async () => {
|
|
198
|
+
// Fill testRequests.json with 20 pending requests, all for one URL, to reach the cap.
|
|
199
|
+
const filler = Array.from({length: 20}, (_, i) => ({
|
|
200
|
+
timeStamp: '260101T0000', description: `Filler Page ${i}`, reason: 'Filler reason.'
|
|
201
|
+
}));
|
|
202
|
+
await fs.writeFile(testRequestsPath, JSON.stringify({'https://example.com/filler': filler}));
|
|
203
|
+
const body = await response(['One Too Many Page', 'https://example.com/one-too-many', 'A reason that is long enough.']);
|
|
204
|
+
const details = body['response content']['details about your request'] as any;
|
|
205
|
+
assert.equal(details.error, undefined);
|
|
206
|
+
const disposition = body['response content']['disposition of your request'] as any;
|
|
207
|
+
assert.ok(disposition['what happens next'].includes('too many requests are awaiting approval right now.'));
|
|
208
|
+
assert.ok(disposition['what happens next'].includes('https://github.com/jrpool/kilotest/issues'));
|
|
209
|
+
assert.ok(disposition['what happens next'].includes('info@kilotest.com'));
|
|
210
|
+
});
|