@jrpool/kilotest 59.1.4 → 60.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -5
- package/api/awaitTest.test.ts +9 -9
- package/api/awaitTest.ts +12 -12
- package/api/getReport.test.ts +4 -4
- package/api/getReport.ts +9 -9
- package/api/listDiagnoses.test.ts +14 -14
- package/api/listDiagnoses.ts +8 -8
- package/api/listIssues.test.ts +15 -15
- package/api/listIssues.ts +18 -20
- package/api/listPages.test.ts +2 -2
- package/api/listPages.ts +4 -4
- package/api/listViolators.test.ts +10 -10
- package/api/listViolators.ts +10 -10
- package/api/orderNewTest.test.ts +1 -2
- package/api/orderNewTest.ts +3 -4
- package/api/orderRetest.test.ts +10 -10
- package/api/orderRetest.ts +13 -14
- package/api/requestRetest.test.ts +11 -11
- package/api/requestRetest.ts +11 -11
- package/api/routes.ts +9 -11
- package/api/schemas.ts +18 -28
- package/api/util.test.ts +2 -2
- package/api/util.ts +7 -7
- package/capability.md +4 -4
- package/docs/AI-TOOLS.md +41 -1
- package/docs/decisions/009-report-id.md +16 -0
- package/index.test.ts +91 -41
- package/index.ts +66 -15
- package/lhm.plugin.json +33 -119
- package/llms-full.txt +18 -24
- package/llms.txt +12 -12
- package/mcp.test.ts +11 -20
- package/mcp.ts +18 -18
- package/openapi-3.0.yaml +1847 -0
- package/openapi.yaml +38 -97
- package/package.json +1 -1
- package/routing.test.ts +1 -1
- package/scripts/generate-openapi.ts +83 -1
- package/server.json +1 -1
- package/util.test.ts +32 -33
- package/util.ts +47 -43
- package/web/hideReportForm/index.ts +5 -6
- package/web/listDiagnoses/index.ts +4 -5
- package/web/listDiagnoses/listDiagnoses.test.ts +13 -13
- package/web/listIssues/index.ts +11 -12
- package/web/listIssues/listIssues.test.ts +24 -24
- package/web/listPages/index.ts +2 -2
- package/web/listPages/listPages.test.ts +2 -2
- package/web/listTopIssues/index.ts +2 -2
- package/web/listViolators/index.ts +5 -6
- package/web/listViolators/listViolators.test.ts +12 -12
- package/web/reannotate/index.ts +3 -3
- package/web/reannotateForm/index.ts +1 -2
- package/web/reportDeletion.ts +9 -12
- package/web/requestRetest/index.ts +2 -2
- package/web/requestRetest/requestRetest.test.ts +7 -7
- package/web/requestRetestForm/index.html +2 -2
- package/web/requestRetestForm/index.ts +3 -4
- package/web/requestRetestForm/requestRetestForm.test.ts +7 -8
- package/web/unhideReportForm/index.ts +4 -6
- package/web/util.test.ts +29 -29
- package/web/util.ts +1 -1
package/api/requestRetest.ts
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
|
|
8
8
|
import {z} from 'zod';
|
|
9
9
|
import {buildRequestDisposition, getResponseMetadata, getThisHost, getToolsFacts} from './util.ts';
|
|
10
|
-
import {checkLength,
|
|
10
|
+
import {checkLength, isReportID, processTestRequest} from '../util.ts';
|
|
11
11
|
import {requestRetestResponseSchema} from './schemas.ts';
|
|
12
12
|
|
|
13
13
|
// TYPES
|
|
@@ -19,16 +19,16 @@ type ResponseContent = z.infer<typeof requestRetestResponseSchema>['response con
|
|
|
19
19
|
|
|
20
20
|
// Returns the response body.
|
|
21
21
|
export const response = async (args: string[]) => {
|
|
22
|
-
const [
|
|
22
|
+
const [reportID = '', reason = ''] = args;
|
|
23
23
|
const thisHost = getThisHost();
|
|
24
24
|
const reasonCheck = checkLength(reason, 20, 100, 'reason');
|
|
25
25
|
// Initialize the response-content properties.
|
|
26
26
|
let requestDetails: ResponseContent['details about your request'];
|
|
27
27
|
let requestDisposition: ResponseContent['disposition of your request'] = null;
|
|
28
|
-
// If the
|
|
29
|
-
if (!
|
|
28
|
+
// If the report identifier is not syntactically valid:
|
|
29
|
+
if (!isReportID(reportID)) {
|
|
30
30
|
requestDetails = {
|
|
31
|
-
error: 'request invalid: the
|
|
31
|
+
error: 'request invalid: the reportID is malformed'
|
|
32
32
|
};
|
|
33
33
|
}
|
|
34
34
|
// Otherwise, if the encoded reason is too short or too long:
|
|
@@ -40,7 +40,7 @@ export const response = async (args: string[]) => {
|
|
|
40
40
|
// Otherwise, i.e. if the request is facially valid:
|
|
41
41
|
else {
|
|
42
42
|
// Process the request. This resolves the cited report's description and URL itself.
|
|
43
|
-
const processResult = await processTestRequest(reason, {
|
|
43
|
+
const processResult = await processTestRequest(reason, {reportID});
|
|
44
44
|
// If the cited report does not exist:
|
|
45
45
|
if (processResult.result === 'nonreport') {
|
|
46
46
|
requestDetails = {
|
|
@@ -81,9 +81,9 @@ export const response = async (args: string[]) => {
|
|
|
81
81
|
'tool collection': getToolsFacts(),
|
|
82
82
|
'tool name': 'requestRetest',
|
|
83
83
|
'this request': {
|
|
84
|
-
description: 'Process my request to retest a page. The
|
|
84
|
+
description: 'Process my request to retest a page. The reportID parameter identifies the latest available report about the page. It was in the response to my earlier listIssues request. The reason property of the request body is the reason why the page should be retested.',
|
|
85
85
|
method: 'POST',
|
|
86
|
-
URL: `${thisHost}/api/requestRetest/${
|
|
86
|
+
URL: `${thisHost}/api/requestRetest/${reportID}`,
|
|
87
87
|
body: {
|
|
88
88
|
reason
|
|
89
89
|
},
|
|
@@ -91,12 +91,12 @@ export const response = async (args: string[]) => {
|
|
|
91
91
|
'tool name': 'listIssues',
|
|
92
92
|
description: 'Provide details about one report, including basics about the issues reported in it.',
|
|
93
93
|
method: 'GET',
|
|
94
|
-
URL: `${thisHost}/api/listIssues/${
|
|
94
|
+
URL: `${thisHost}/api/listIssues/${reportID}`
|
|
95
95
|
}
|
|
96
96
|
},
|
|
97
97
|
'URLs of similar requests for web users': {
|
|
98
|
-
'this request': `${thisHost}/requestRetestForm.html/${
|
|
99
|
-
'closest ancestor request': `${thisHost}/listIssues.html/${
|
|
98
|
+
'this request': `${thisHost}/requestRetestForm.html/${reportID}`,
|
|
99
|
+
'closest ancestor request': `${thisHost}/listIssues.html/${reportID}`
|
|
100
100
|
},
|
|
101
101
|
'response metadata': getResponseMetadata(),
|
|
102
102
|
'response content': responseContent
|
package/api/routes.ts
CHANGED
|
@@ -43,7 +43,7 @@ export const routes = [
|
|
|
43
43
|
{
|
|
44
44
|
operationId: 'listIssues',
|
|
45
45
|
method: 'get',
|
|
46
|
-
path: '/api/listIssues/{
|
|
46
|
+
path: '/api/listIssues/{reportID}',
|
|
47
47
|
summary: 'List basics about the issues in one report',
|
|
48
48
|
pathParamsSchema: z.object(listIssuesSchema),
|
|
49
49
|
responseSchema: listIssuesResponseSchema
|
|
@@ -51,7 +51,7 @@ export const routes = [
|
|
|
51
51
|
{
|
|
52
52
|
operationId: 'listViolators',
|
|
53
53
|
method: 'get',
|
|
54
|
-
path: '/api/listViolators/{issueID}/{
|
|
54
|
+
path: '/api/listViolators/{issueID}/{reportID}',
|
|
55
55
|
summary: 'List elements reported as exhibiting one issue',
|
|
56
56
|
pathParamsSchema: z.object(listViolatorsSchema),
|
|
57
57
|
responseSchema: listViolatorsResponseSchema
|
|
@@ -59,7 +59,7 @@ export const routes = [
|
|
|
59
59
|
{
|
|
60
60
|
operationId: 'listDiagnoses',
|
|
61
61
|
method: 'get',
|
|
62
|
-
path: '/api/listDiagnoses/{catalogIndex}/{issueID}/{
|
|
62
|
+
path: '/api/listDiagnoses/{catalogIndex}/{issueID}/{reportID}',
|
|
63
63
|
summary: 'List diagnoses of how one element exhibited one issue',
|
|
64
64
|
pathParamsSchema: z.object(listDiagnosesSchema),
|
|
65
65
|
responseSchema: listDiagnosesResponseSchema
|
|
@@ -67,7 +67,7 @@ export const routes = [
|
|
|
67
67
|
{
|
|
68
68
|
operationId: 'getReport',
|
|
69
69
|
method: 'get',
|
|
70
|
-
path: '/api/getReport/{
|
|
70
|
+
path: '/api/getReport/{reportID}',
|
|
71
71
|
summary: 'Get one full report in JSON',
|
|
72
72
|
pathParamsSchema: z.object(getReportSchema),
|
|
73
73
|
responseSchema: getReportResponseSchema
|
|
@@ -83,11 +83,10 @@ export const routes = [
|
|
|
83
83
|
{
|
|
84
84
|
operationId: 'requestRetest',
|
|
85
85
|
method: 'post',
|
|
86
|
-
path: '/api/requestRetest/{
|
|
86
|
+
path: '/api/requestRetest/{reportID}',
|
|
87
87
|
summary: 'Request that a page be retested',
|
|
88
88
|
pathParamsSchema: z.object({
|
|
89
|
-
|
|
90
|
-
jobID: requestRetestSchema.jobID
|
|
89
|
+
reportID: requestRetestSchema.reportID
|
|
91
90
|
}),
|
|
92
91
|
bodySchema: z.object({reason: requestRetestSchema.reason}),
|
|
93
92
|
responseSchema: requestRetestResponseSchema
|
|
@@ -111,11 +110,10 @@ export const routes = [
|
|
|
111
110
|
{
|
|
112
111
|
operationId: 'orderRetest',
|
|
113
112
|
method: 'post',
|
|
114
|
-
path: '/api/orderRetest/{
|
|
113
|
+
path: '/api/orderRetest/{reportID}',
|
|
115
114
|
summary: 'Order that a page be retested immediately, without manual approval',
|
|
116
115
|
pathParamsSchema: z.object({
|
|
117
|
-
|
|
118
|
-
jobID: orderRetestSchema.jobID
|
|
116
|
+
reportID: orderRetestSchema.reportID
|
|
119
117
|
}),
|
|
120
118
|
bodySchema: z.object({reason: orderRetestSchema.reason}),
|
|
121
119
|
responseSchema: orderRetestResponseSchema
|
|
@@ -123,7 +121,7 @@ export const routes = [
|
|
|
123
121
|
{
|
|
124
122
|
operationId: 'awaitTest',
|
|
125
123
|
method: 'post',
|
|
126
|
-
path: '/api/awaitTest/{
|
|
124
|
+
path: '/api/awaitTest/{reportID}',
|
|
127
125
|
summary: 'Wait for a job ordered via orderNewTest or orderRetest to complete',
|
|
128
126
|
pathParamsSchema: z.object(awaitTestSchema),
|
|
129
127
|
// No caller-supplied fields beyond the path parameters; an empty body schema (rather
|
package/api/schemas.ts
CHANGED
|
@@ -14,31 +14,27 @@ type ZodTypeAny = import('zod').ZodTypeAny;
|
|
|
14
14
|
|
|
15
15
|
// REQUEST SCHEMAS
|
|
16
16
|
|
|
17
|
-
// listIssues: GET /api/listIssues/{
|
|
17
|
+
// listIssues: GET /api/listIssues/{reportID}
|
|
18
18
|
export const listIssuesSchema = {
|
|
19
|
-
|
|
20
|
-
jobID: z.string().describe('Job identifier of the report (example: x9z)')
|
|
19
|
+
reportID: z.string().describe('Identifier of the report (example: 260503T0432-x9z)')
|
|
21
20
|
};
|
|
22
21
|
|
|
23
|
-
// listViolators: GET /api/listViolators/{issueID}/{
|
|
22
|
+
// listViolators: GET /api/listViolators/{issueID}/{reportID}
|
|
24
23
|
export const listViolatorsSchema = {
|
|
25
24
|
issueID: z.string().describe('Issue identifier (example: contrastPoor)'),
|
|
26
|
-
|
|
27
|
-
jobID: z.string().describe('Job identifier of the report (example: x9z)')
|
|
25
|
+
reportID: z.string().describe('Identifier of the report (example: 260503T0432-x9z)')
|
|
28
26
|
};
|
|
29
27
|
|
|
30
|
-
// listDiagnoses: GET /api/listDiagnoses/{catalogIndex}/{issueID}/{
|
|
28
|
+
// listDiagnoses: GET /api/listDiagnoses/{catalogIndex}/{issueID}/{reportID}
|
|
31
29
|
export const listDiagnosesSchema = {
|
|
32
30
|
catalogIndex: z.string().describe('Identifier of the issue-exhibiting element in the catalog of elements on the page (example: 372)'),
|
|
33
31
|
issueID: z.string().describe('Issue identifier (example: contrastPoor)'),
|
|
34
|
-
|
|
35
|
-
jobID: z.string().describe('Job identifier of the report (example: x9z)')
|
|
32
|
+
reportID: z.string().describe('Identifier of the report (example: 260503T0432-x9z)')
|
|
36
33
|
};
|
|
37
34
|
|
|
38
|
-
// getReport: GET /api/getReport/{
|
|
35
|
+
// getReport: GET /api/getReport/{reportID}
|
|
39
36
|
export const getReportSchema = {
|
|
40
|
-
|
|
41
|
-
jobID: z.string().describe('Job identifier of the report (example: x9z)')
|
|
37
|
+
reportID: z.string().describe('Identifier of the report (example: 260503T0432-x9z)')
|
|
42
38
|
};
|
|
43
39
|
|
|
44
40
|
// requestNewTest: POST /api/requestNewTest
|
|
@@ -48,10 +44,9 @@ export const requestNewTestSchema = {
|
|
|
48
44
|
reason: z.string().describe('20- to 100-character reason why the page should be tested')
|
|
49
45
|
};
|
|
50
46
|
|
|
51
|
-
// requestRetest: POST /api/requestRetest/{
|
|
47
|
+
// requestRetest: POST /api/requestRetest/{reportID}
|
|
52
48
|
export const requestRetestSchema = {
|
|
53
|
-
|
|
54
|
-
jobID: z.string().describe('Job identifier of the latest report about the page (example: x9z)'),
|
|
49
|
+
reportID: z.string().describe('Identifier of the latest report about the page (example: 260503T0432-x9z)'),
|
|
55
50
|
reason: z.string().describe('20- to 100-character reason why the page should be retested')
|
|
56
51
|
};
|
|
57
52
|
|
|
@@ -67,17 +62,15 @@ export const orderNewTestSchema = {
|
|
|
67
62
|
reason: z.string().describe('20- to 100-character reason why the page should be tested')
|
|
68
63
|
};
|
|
69
64
|
|
|
70
|
-
// orderRetest: POST /api/orderRetest/{
|
|
65
|
+
// orderRetest: POST /api/orderRetest/{reportID}
|
|
71
66
|
export const orderRetestSchema = {
|
|
72
|
-
|
|
73
|
-
jobID: z.string().describe('Job identifier of the latest report about the page (example: x9z)'),
|
|
67
|
+
reportID: z.string().describe('Identifier of the latest report about the page (example: 260503T0432-x9z)'),
|
|
74
68
|
reason: z.string().describe('20- to 100-character reason why the page should be retested')
|
|
75
69
|
};
|
|
76
70
|
|
|
77
|
-
// awaitTest: POST /api/awaitTest
|
|
71
|
+
// awaitTest: POST /api/awaitTest/{reportID}
|
|
78
72
|
export const awaitTestSchema = {
|
|
79
|
-
|
|
80
|
-
jobID: z.string().describe('Job identifier of the ordered job, from the report identifier returned by orderNewTest (example: x9z)')
|
|
73
|
+
reportID: z.string().describe('Report identifier returned by orderNewTest or orderRetest (example: 260503T0432-x9z)')
|
|
81
74
|
};
|
|
82
75
|
|
|
83
76
|
// listPages takes no input; omitted (mcp.ts already uses inputSchema: {}).
|
|
@@ -164,7 +157,7 @@ const responseMetadataSchema = z.object({
|
|
|
164
157
|
|
|
165
158
|
// Facts about a report, as constructed by Kilotest's own getReportBasics (never copied from a Testaro report).
|
|
166
159
|
const reportBasicsSchema = z.object({
|
|
167
|
-
identifier: z.string().describe('
|
|
160
|
+
identifier: z.string().describe('Identifier of the report (example: 260503T0432-x9z).'),
|
|
168
161
|
'completion date and time': z.string(),
|
|
169
162
|
'days since the report was completed': z.number().nullable(),
|
|
170
163
|
'tested web page': z.object({description: z.string(), URL: z.string()}),
|
|
@@ -196,7 +189,7 @@ const issueBasicsOrErrorSchema = z.union([
|
|
|
196
189
|
|
|
197
190
|
// Facts about a report among those about one page.
|
|
198
191
|
const reportTimeSchema = z.object({
|
|
199
|
-
identifier: z.string().describe('
|
|
192
|
+
identifier: z.string().describe('Identifier of the report (example: 260503T0432-x9z).'),
|
|
200
193
|
'completion date and time': z.string(),
|
|
201
194
|
'days since the report was completed': z.number().nullable()
|
|
202
195
|
});
|
|
@@ -367,10 +360,7 @@ export const requestFeatureResponseSchema = envelope(
|
|
|
367
360
|
);
|
|
368
361
|
|
|
369
362
|
// Identifies the report a successfully ordered or awaited job will produce, once complete.
|
|
370
|
-
const reportIdentifierSchema = z.
|
|
371
|
-
timeStamp: z.string().describe('Timestamp to pass to listIssues once the report is ready.'),
|
|
372
|
-
jobID: z.string().describe('Job identifier to pass to listIssues once the report is ready.')
|
|
373
|
-
}).meta({id: 'ReportIdentifier'});
|
|
363
|
+
const reportIdentifierSchema = z.string().describe('Identifier of the report, to pass to listIssues once the report is ready (example: 260503T0432-x9z).');
|
|
374
364
|
|
|
375
365
|
export const orderNewTestResponseSchema = envelope(
|
|
376
366
|
'POST',
|
|
@@ -443,7 +433,7 @@ export const awaitTestResponseSchema = envelope(
|
|
|
443
433
|
'closest ancestor request': z.null()
|
|
444
434
|
}),
|
|
445
435
|
// awaitTest can be reached from either orderNewTest or orderRetest, and its own
|
|
446
|
-
//
|
|
436
|
+
// reportID parameter does not say which one produced it, so both possible
|
|
447
437
|
// ancestors are reported, rather than arbitrarily naming just one as "the" ancestor.
|
|
448
438
|
z.array(requestReferenceSchema).length(2)
|
|
449
439
|
);
|
package/api/util.test.ts
CHANGED
|
@@ -74,7 +74,7 @@ test('getIssueSpec returns null for the ignorable issue ID', () => {
|
|
|
74
74
|
|
|
75
75
|
test('getReportBasics returns an error for a nonexistent report', async () => {
|
|
76
76
|
process.env.DB_DIR = (await import('../test/dbFixture.ts')).fixtureDBDir;
|
|
77
|
-
const basics: any = await getReportBasics('999999T9999
|
|
77
|
+
const basics: any = await getReportBasics('999999T9999-xyz');
|
|
78
78
|
assert.ok(basics.error);
|
|
79
79
|
if (savedDBDir !== undefined) {
|
|
80
80
|
process.env.DB_DIR = savedDBDir;
|
|
@@ -91,7 +91,7 @@ test('getReportBasics returns an error for a report whose extract cannot be read
|
|
|
91
91
|
const reportPath = path.join(process.env.DB_DIR, 'reports', '888888T8888-bad.json');
|
|
92
92
|
await fs.writeFile(reportPath, '{"malformed": true}\n');
|
|
93
93
|
try {
|
|
94
|
-
const basics: any = await getReportBasics('888888T8888
|
|
94
|
+
const basics: any = await getReportBasics('888888T8888-bad');
|
|
95
95
|
assert.ok(basics.error);
|
|
96
96
|
}
|
|
97
97
|
finally {
|
package/api/util.ts
CHANGED
|
@@ -133,20 +133,20 @@ export const getRuleEnginesFacts = (ruleEngineIDSet: Iterable<string>) => {
|
|
|
133
133
|
};
|
|
134
134
|
// Returns the basics about a report, or an error object if the report cannot be read.
|
|
135
135
|
export const getReportBasics = async (
|
|
136
|
-
|
|
136
|
+
reportID: string
|
|
137
137
|
): Promise<ReportBasics | {error: string}> => {
|
|
138
138
|
// Verify that the report exists.
|
|
139
|
-
const reportStats = await getReportStats(
|
|
139
|
+
const reportStats = await getReportStats(reportID);
|
|
140
140
|
// If it does not exist:
|
|
141
141
|
if (!reportStats) {
|
|
142
142
|
// Log and return this.
|
|
143
|
-
console.error(`Report ${
|
|
143
|
+
console.error(`Report ${reportID} does not exist.`);
|
|
144
144
|
return {
|
|
145
|
-
error: `Report ${
|
|
145
|
+
error: `Report ${reportID} could not be retrieved.`
|
|
146
146
|
};
|
|
147
147
|
}
|
|
148
148
|
// Otherwise, i.e. if it exists, get an extract of the report.
|
|
149
|
-
const extract = await getReportExtract(
|
|
149
|
+
const extract = await getReportExtract(reportID);
|
|
150
150
|
// If this failed, return why.
|
|
151
151
|
if ('error' in extract) {
|
|
152
152
|
return extract;
|
|
@@ -154,10 +154,10 @@ export const getReportBasics = async (
|
|
|
154
154
|
const {url, description, reportTime} = extract;
|
|
155
155
|
// Get whether this report has been superseded.
|
|
156
156
|
const isSuperseded = (await getReportExtracts(true))
|
|
157
|
-
.every(ex => ex.
|
|
157
|
+
.every(ex => ex.id !== reportID);
|
|
158
158
|
// Get the basics about the report.
|
|
159
159
|
const basics = {
|
|
160
|
-
identifier:
|
|
160
|
+
identifier: reportID,
|
|
161
161
|
'completion date and time': reportTime,
|
|
162
162
|
'days since the report was completed': getAgoDays(new Date(reportTime)),
|
|
163
163
|
'tested web page': {
|
package/capability.md
CHANGED
|
@@ -6,7 +6,7 @@ Web-page front-end quality audit. Kilotest tests web pages for front-end quality
|
|
|
6
6
|
|
|
7
7
|
## Purpose
|
|
8
8
|
|
|
9
|
-
Language models and AI platforms often need accurate, page-specific data about the front-end quality of a web page to answer user questions.
|
|
9
|
+
Language models and AI platforms often need accurate, page-specific data about the front-end quality of a web page to answer user questions. Any single rule engine, such as `axe-core`, `QualWeb`, or `WAVE`, provides such data, but with a large coverage gap. Orchestrating and reconciling numerous complementary rule engines would be difficult. Kilotest solves that problem by testing pages with an ensemble of numerous independent rule engines, making their results comparable, saving the results as JSON reports, and mining the reports for facts at selectable levels of detail.
|
|
10
10
|
|
|
11
11
|
## Interfaces
|
|
12
12
|
|
|
@@ -18,11 +18,11 @@ Language models and AI platforms often need accurate, page-specific data about t
|
|
|
18
18
|
|
|
19
19
|
## Initial input
|
|
20
20
|
|
|
21
|
-
A request for a list of
|
|
21
|
+
A request for a list of tested pages.
|
|
22
22
|
|
|
23
23
|
## Initial output
|
|
24
24
|
|
|
25
|
-
A list of
|
|
25
|
+
A list of tested pages, with URLs for requesting further details about their test results.
|
|
26
26
|
|
|
27
27
|
## Subsequent inputs
|
|
28
28
|
|
|
@@ -34,7 +34,7 @@ A list of available reports.
|
|
|
34
34
|
|
|
35
35
|
- From a **request** for a new test or a retest: if valid, confirmation of receipt. If the request is manager-approved and executed, the list of available reports will include the new report, typically in 1 hour to 1 day.
|
|
36
36
|
- From an **order** for a new test or a retest: if valid, confirmation of receipt and immediate approval or rejection. If the request is approved, the output will include the identifier of the forthcoming report and two methods for getting the report: (1) allowing time (2 to 4 minutes) for completion of the report and then requesting it, and (2) making a request whose response will await the report and then link to it. Orders are enabled in the MCP and API interfaces only.
|
|
37
|
-
- From a request for findings: Details about issues reported, related WCAG standards, what rules were violated, and which DOM elements violated them.
|
|
37
|
+
- From a request for findings: Details at several granularities about issues reported, related WCAG standards, what rules were violated, and which DOM elements violated them.
|
|
38
38
|
- From complaints and suggestions about features: acknowledgment of receipt and of the manager being notified.
|
|
39
39
|
|
|
40
40
|
## Side effects
|
package/docs/AI-TOOLS.md
CHANGED
|
@@ -65,9 +65,49 @@ The external features that support the use of Kilotest as a collection of AI too
|
|
|
65
65
|
Releasing a new version is a two-script procedure:
|
|
66
66
|
|
|
67
67
|
- On a feature branch, commit all substantive changes, then execute `npm run prePR -- <level>` (`major`, `minor`, or `patch`). The script bumps `package.json` and `package-lock.json`, regenerates `openapi.yaml` and the versions of `server.json` and `lhm.plugin.json` via the `version` hook, commits and pushes the result, and records the branch name for the next step.
|
|
68
|
-
- Submit a pull request and merge it to `main` (triggers `publish-mcp.yml`
|
|
68
|
+
- Submit a pull request and merge it to `main` (triggers `publish-mcp.yml`, which publishes `server.json` to the MCP Registry).
|
|
69
69
|
- Execute `npm run postPR`. The script checks out and pulls `main`, verifies that the merge landed, publishes to npm (whose `postpublish` hook updates the LobeHub listing), deletes the feature branch, and prunes remote-tracking branches. If the LobeHub update fails, retry it alone with `npm run publish:lhm`.
|
|
70
70
|
|
|
71
|
+
### Contract changes and external registries
|
|
72
|
+
|
|
73
|
+
A change to the tool or API contract (tool names, parameter names, schemas, response shapes, or paths) makes every stored copy of the contract stale. Which copies exist and what to do about each:
|
|
74
|
+
|
|
75
|
+
#### In-repository contract copies
|
|
76
|
+
|
|
77
|
+
- `openapi.yaml`: regenerated automatically by the `version` hook during `npm run prePR`; never edit it by hand.
|
|
78
|
+
- `openapi-3.0.yaml`: an OpenAPI 3.0.3 conversion of `openapi.yaml`, regenerated by the same hook, for registries whose importers reject 3.1 documents; never edit it by hand.
|
|
79
|
+
- `server.json`: contains only the server descriptor (name, description, repository, remotes, version), no tool schemas. Its version is synced by `update-versions.ts`; edit it manually only if the server description or remote URL changes.
|
|
80
|
+
- `lhm.plugin.json`: embeds every tool’s input and output schema and its descriptions, in the LobeHub manifest format. It is hand-maintained, not generated, so transform it to match `api/schemas.ts` and `mcp.ts` whenever the contract changes.
|
|
81
|
+
- `llms.txt`, `llms-full.txt`, `capability.md`, `README.md`: hand-maintained; search them for stale parameter and path names after a contract change.
|
|
82
|
+
|
|
83
|
+
#### External registries that update themselves, needing no maintainer action
|
|
84
|
+
|
|
85
|
+
- The official MCP Registry: `publish-mcp.yml` republishes `server.json` whenever `server.json` or `api/version.ts` changes on `main`, i.e. on every release. The listing contains no tool schemas, so a contract change alone needs nothing extra.
|
|
86
|
+
- PulseMCP and the VS Code MCP gallery: crawl the MCP Registry.
|
|
87
|
+
- Smithery and Glama: derive their tool listings from the deployed server’s own `tools/list` response (and repository scans) on their own crawl schedules. Their listings can lag temporarily behind a contract change, but they correct themselves.
|
|
88
|
+
- `openapi-directory` (APIs.guru): maintains its own copy of `openapi.yaml`, refreshed from the registered source URL on its own schedule.
|
|
89
|
+
- `public-apis`, the WAI evaluation tools list, Awesome MCP Servers, and the Claude connector directory: store only a link or a connector name and remote URL; contract changes do not affect them.
|
|
90
|
+
|
|
91
|
+
#### External registries requiring maintainer action
|
|
92
|
+
|
|
93
|
+
- The LobeHub MCP Market listing is published from `lhm.plugin.json` and is non-automated, because LobeHub requires browser-based OAuth. A normal release pushes the updated file via the `postpublish` hook; to push a contract update without a release, run `npm run publish:lhm`.
|
|
94
|
+
- The RapidAPI Hub listing stores a copy of the API endpoints. Its importer accepts only OpenAPI 3.0 documents, so after releasing a contract change, update the definition in Studio: `My APIs` → the Kilotest API project → `Hub Listing` → `Definitions` → `CI/CD` → `Import OpenAPI`, and upload the regenerated `openapi-3.0.yaml` (not `openapi.yaml`, which is 3.1 and is rejected). The `/publish/…` pages contain only listing metadata, not the spec.
|
|
95
|
+
|
|
96
|
+
#### Step by step
|
|
97
|
+
|
|
98
|
+
When you make an API or tool contract change in a feature branch, here are all the maintenance steps:
|
|
99
|
+
|
|
100
|
+
1. `lhm.plugin.json`: Transform it to match `api/schemas.ts` and `mcp.ts`.
|
|
101
|
+
2. `llms.txt`, `llms-full.txt`, `capability.md`, `README.md`: Update parameter and path names.
|
|
102
|
+
3. Stage and commit all changes.
|
|
103
|
+
4. Execute `npm run prePR -- <level>` (`major`, `minor`, or `patch`).
|
|
104
|
+
5. Submit a pull request.
|
|
105
|
+
6. When all checks succeed, squash-merge to `main`.
|
|
106
|
+
7. Execute `npm run postPR` (with interactive browser OAuth).
|
|
107
|
+
8. Visit the [RapidAPI Studio](https://rapidapi.com/studio) and update the API definition at `Hub Listing` → `Definitions` → `CI/CD` → `Import OpenAPI` by uploading the regenerated `openapi-3.0.yaml`.
|
|
108
|
+
|
|
109
|
+
If any step fails, see the details above for troubleshooting instructions.
|
|
110
|
+
|
|
71
111
|
## Use cases
|
|
72
112
|
|
|
73
113
|
The rationale for Kilotest as a collection of tools for language models is set forth in the `llms-full.txt` file and is not repeated here.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
---
|
|
2
|
+
date: 2026-10-17
|
|
3
|
+
status: accepted
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Report identifier
|
|
7
|
+
|
|
8
|
+
## Context, problem, and decision
|
|
9
|
+
|
|
10
|
+
Kilotest identifies each report with a string composed of the creation time stamp of the job that produced the report, in `YYMMDDTHHmm` format, and a random alphanumeric suffix, joined with a hyphen (example: `260503T0432-x9z`). The storage layer already treats this composed string as a unit: it is the report filename stem, the job identifier, and the value emitted as the report `identifier` in API responses.
|
|
11
|
+
|
|
12
|
+
Formerly, however, the request surfaces treated the identifier as two parameters, `timeStamp` and `jobID`, so that URLs had the form `/api/listIssues/{timeStamp}/{jobID}` and MCP tools took separate `timeStamp` and `jobID` inputs. The two components were never decomposed for separate semantic use: every split site recombined them immediately or validated them separately for no benefit. The split representation therefore added parameters, path segments, and boilerplate without adding meaning.
|
|
13
|
+
|
|
14
|
+
The identifier is now consistently treated as a unit named `reportID`, validated by a single `isReportID` predicate. Routes that had `{timeStamp}` and `{jobID}` path parameters now have one `{reportID}` parameter. The name `reportID`, rather than `jobID`, is used because a job is ephemeral (typically a few minutes), while a report lives indefinitely and is consulted repeatedly.
|
|
15
|
+
|
|
16
|
+
Because human users may have bookmarked web URLs containing the two-segment tail, legacy web paths redirect permanently (301 for GET, 308 for POST) to the single-parameter URLs, preserving query strings. The REST API and the MCP interface had no known consumers when this change was made, so they were cut over without transitional compatibility; the external registry listings (`openapi.yaml`, `server.json`, `lhm.plugin.json`) were updated quasi-simultaneously.
|