@jrpool/kilotest 46.0.8 → 46.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.devin/config.local.json +3 -1
- package/.github/workflows/{smoke-test.yml → ci.yml} +8 -2
- package/alerts.test.js +137 -0
- package/api/listDiagnoses.test.js +44 -0
- package/api/listIssues.js +1 -1
- package/api/listIssues.test.js +10 -0
- package/api/listReports.test.js +6 -2
- package/api/listViolators.test.js +35 -0
- package/api/requestTest.test.js +7 -0
- package/api/routes.test.js +63 -0
- package/api/util.js +8 -10
- package/api/util.test.js +68 -2
- package/apiIntegration.test.js +13 -3
- package/docs/{Smoke Test.json → CI.json} +4 -3
- package/index.js +64 -77
- package/index.test.js +1152 -0
- package/indexBalances.test.js +188 -0
- package/indexFallback.test.js +33 -0
- package/indexServe.test.js +59 -0
- package/indexWorkers.test.js +79 -0
- package/mcp.js +2 -2
- package/mcp.test.js +281 -0
- package/package.json +10 -5
- package/routing.test.js +13 -3
- package/test/fixtures/buildFixtures.js +83 -0
- package/test/fixtures/db/reports/260101T0008-mul.json +137 -0
- package/test/fixtures/db/reports/260101T0009-brd.json +147 -0
- package/test/setup.cjs +14 -0
- package/util.js +16 -6
- package/util.test.js +886 -3
- package/web/ai0BalanceForm/ai0BalanceForm.test.js +88 -0
- package/web/expungeReportsForm/expungeReportsForm.test.js +114 -0
- package/web/hideReportForm/hideReportForm.test.js +82 -0
- package/web/listDiagnoses/listDiagnoses.test.js +160 -0
- package/web/listIssues/listIssues.test.js +109 -0
- package/web/listReports/listReports.test.js +139 -3
- package/web/listRules/listRules.test.js +103 -0
- package/web/listTopIssues/listTopIssues.test.js +126 -0
- package/web/listViolators/index.js +1 -2
- package/web/listViolators/listViolators.test.js +256 -0
- package/web/pruneReportsForm/pruneReportsForm.test.js +86 -0
- package/web/reannotate/reannotate.test.js +144 -0
- package/web/reannotateForm/reannotateForm.test.js +134 -0
- package/web/renewWCAG/renewWCAG.test.js +124 -0
- package/web/requestRetest/requestRetest.test.js +119 -0
- package/web/requestRetestForm/requestRetestForm.test.js +62 -0
- package/web/requestTest/requestTest.test.js +115 -0
- package/web/requestTestForm/requestTestForm.test.js +40 -0
- package/web/rewindReportsForm/rewindReportsForm.test.js +86 -0
- package/web/tutorial/tutorial.test.js +97 -0
- package/web/unhideReportForm/unhideReportForm.test.js +135 -0
package/index.js
CHANGED
|
@@ -98,23 +98,7 @@ const routes = exports.routes = {
|
|
|
98
98
|
'/tutorialComment.html'
|
|
99
99
|
]
|
|
100
100
|
};
|
|
101
|
-
// Returns whether a pathname matches a glob-style pattern.
|
|
102
|
-
const matchPath = (pattern, pathname) => {
|
|
103
|
-
const regex = new RegExp(
|
|
104
|
-
'^' + pattern.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*') + '$'
|
|
105
|
-
);
|
|
106
|
-
return regex.test(pathname);
|
|
107
|
-
};
|
|
108
|
-
// Returns whether a pathname is authorized for a method.
|
|
109
|
-
const isPathAllowed = (method, pathname) => {
|
|
110
|
-
const patterns = routes[method] || [];
|
|
111
|
-
return patterns.some(pattern => matchPath(pattern, pathname));
|
|
112
|
-
};
|
|
113
|
-
|
|
114
101
|
const protocol = process.env.PROTOCOL || 'http';
|
|
115
|
-
const queuePath = () => path.join(jobsPath(), 'queue');
|
|
116
|
-
const claimedPath = () => path.join(jobsPath(), 'claimed');
|
|
117
|
-
const failedPath = () => path.join(jobsPath(), 'failed');
|
|
118
102
|
// Credentials of the Testaro workers, by worker ID, from a JSON-object environment variable.
|
|
119
103
|
// Each worker ID maps to a secret (used only to authenticate the worker, never published) and a
|
|
120
104
|
// name (a non-secret label safe to publish, e.g. in report data and logs).
|
|
@@ -137,10 +121,26 @@ const AI_MODEL0_OUTPUT_PRICE = Number(process.env.AI_MODEL0_OUTPUT_PRICE);
|
|
|
137
121
|
|
|
138
122
|
// FUNCTIONS
|
|
139
123
|
|
|
124
|
+
const queuePath = () => path.join(jobsPath(), 'queue');
|
|
125
|
+
const claimedPath = () => path.join(jobsPath(), 'claimed');
|
|
126
|
+
const failedPath = () => path.join(jobsPath(), 'failed');
|
|
127
|
+
// Returns whether a pathname matches a glob-style pattern.
|
|
128
|
+
const matchPath = (pattern, pathname) => {
|
|
129
|
+
const regex = new RegExp(
|
|
130
|
+
'^' + pattern.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*') + '$'
|
|
131
|
+
);
|
|
132
|
+
return regex.test(pathname);
|
|
133
|
+
};
|
|
134
|
+
// Returns whether a pathname is authorized for a method.
|
|
135
|
+
const isPathAllowed = exports.isPathAllowed = (method, pathname) => {
|
|
136
|
+
const patterns = routes[method] || [];
|
|
137
|
+
return patterns.some(pattern => matchPath(pattern, pathname));
|
|
138
|
+
};
|
|
140
139
|
// Serves or sends an error message.
|
|
141
|
-
const serveError = async (error, response, isHumanUser = true, statusCode = 400) => {
|
|
140
|
+
const serveError = exports.serveError = async (error, response, isHumanUser = true, statusCode = 400) => {
|
|
142
141
|
const errorLines = Object.entries(error).map(pair => `${pair[0]}: ${pair[1]}`);
|
|
143
|
-
|
|
142
|
+
const errorSummary = errorLines.join('\n') || 'ERROR';
|
|
143
|
+
console.log(errorSummary);
|
|
144
144
|
if (!response.writableEnded) {
|
|
145
145
|
response.statusCode = statusCode;
|
|
146
146
|
// If the request is from a human user:
|
|
@@ -151,7 +151,8 @@ const serveError = async (error, response, isHumanUser = true, statusCode = 400)
|
|
|
151
151
|
response.setHeader('Access-Control-Allow-Origin', '*');
|
|
152
152
|
response.setHeader('Cache-Control', 'public, max-age=300, stale-while-revalidate=3000');
|
|
153
153
|
const errorTemplate = await fs.readFile('error.html', 'utf8');
|
|
154
|
-
const
|
|
154
|
+
const errorMessage = error.message || 'ERROR';
|
|
155
|
+
const errorPage = errorTemplate.replace(/__error__/, errorMessage);
|
|
155
156
|
response.end(errorPage);
|
|
156
157
|
}
|
|
157
158
|
// Otherwise, i.e. if it is from an agent:
|
|
@@ -207,21 +208,16 @@ const checkBalancesForAlerts = async report => {
|
|
|
207
208
|
const balanceData = JSON.parse(balanceJSON);
|
|
208
209
|
// Get an estimate of the balance after this job.
|
|
209
210
|
const newBalance = balanceData.balance - cost;
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
);
|
|
221
|
-
}
|
|
222
|
-
}
|
|
223
|
-
else {
|
|
224
|
-
console.log('ERROR: AI service 0 balance is not a number');
|
|
211
|
+
// Update the recorded balance.
|
|
212
|
+
await fs.writeFile(balancePath, getJSON({balance: newBalance}));
|
|
213
|
+
console.log(`Estimated new AI Service 0 balance: $${newBalance.toFixed(2)}`);
|
|
214
|
+
// If it is nearing exhaustion:
|
|
215
|
+
if (newBalance < AI_SERVICE0_THRESHOLD) {
|
|
216
|
+
// Alert a manager.
|
|
217
|
+
await sendAlert(
|
|
218
|
+
'Kilotest: AI service 0 balance low',
|
|
219
|
+
`Balance of AI service 0 account (https://console.anthropic.com) only about $${newBalance.toFixed(2)} (about $0.01 used per job)`
|
|
220
|
+
);
|
|
225
221
|
}
|
|
226
222
|
}
|
|
227
223
|
catch (error) {
|
|
@@ -231,9 +227,11 @@ const checkBalancesForAlerts = async report => {
|
|
|
231
227
|
}
|
|
232
228
|
};
|
|
233
229
|
// Creates an error object about a suspicious request.
|
|
234
|
-
const getAbuseError = (request, reason) => {
|
|
230
|
+
const getAbuseError = exports.getAbuseError = (request, reason) => {
|
|
235
231
|
const {method, url, headers} = request;
|
|
236
|
-
const
|
|
232
|
+
const forwardedFor = headers['x-forwarded-for'];
|
|
233
|
+
const remoteAddress = request.socket.remoteAddress;
|
|
234
|
+
const ip = forwardedFor || remoteAddress || 'unknown';
|
|
237
235
|
return {
|
|
238
236
|
message: 'Invalid request',
|
|
239
237
|
reason,
|
|
@@ -253,13 +251,7 @@ const getBasicAuth = request => {
|
|
|
253
251
|
if (!match) {
|
|
254
252
|
return null;
|
|
255
253
|
}
|
|
256
|
-
|
|
257
|
-
try {
|
|
258
|
-
decoded = Buffer.from(match[1], 'base64').toString('utf8');
|
|
259
|
-
}
|
|
260
|
-
catch {
|
|
261
|
-
return null;
|
|
262
|
-
}
|
|
254
|
+
const decoded = Buffer.from(match[1], 'base64').toString('utf8');
|
|
263
255
|
const sepIndex = decoded.indexOf(':');
|
|
264
256
|
if (sepIndex === -1) {
|
|
265
257
|
return null;
|
|
@@ -568,7 +560,8 @@ const requestHandler = async (request, response) => {
|
|
|
568
560
|
'.webp': 'image/webp',
|
|
569
561
|
'.svg': 'image/svg+xml'
|
|
570
562
|
};
|
|
571
|
-
|
|
563
|
+
const mimeType = mimeTypes[ext] || 'application/octet-stream';
|
|
564
|
+
setHeaders(mimeType, null, 'low');
|
|
572
565
|
response.end(img);
|
|
573
566
|
}
|
|
574
567
|
catch {
|
|
@@ -787,8 +780,9 @@ const requestHandler = async (request, response) => {
|
|
|
787
780
|
// Otherwise, if it is report acquisition:
|
|
788
781
|
else if (service === 'report') {
|
|
789
782
|
const {report} = postData;
|
|
790
|
-
const
|
|
791
|
-
const {
|
|
783
|
+
const reportObj = report || {};
|
|
784
|
+
const {id, target} = reportObj;
|
|
785
|
+
const {what, url} = target || {};
|
|
792
786
|
const [timeStamp, jobID] = id?.split('-') ?? ['', ''];
|
|
793
787
|
// If the request is syntactically valid:
|
|
794
788
|
if (id && isTimeStamp(timeStamp) && isJobID(jobID) && what && url) {
|
|
@@ -834,12 +828,6 @@ const requestHandler = async (request, response) => {
|
|
|
834
828
|
await serveError({message: 'ERROR: Request invalid'}, response, false);
|
|
835
829
|
}
|
|
836
830
|
}
|
|
837
|
-
// Otherwise, i.e. if the service is invalid:
|
|
838
|
-
else {
|
|
839
|
-
await serveError(
|
|
840
|
-
{message: 'ERROR: Invalid service request from Testaro worker'}, response, false
|
|
841
|
-
);
|
|
842
|
-
}
|
|
843
831
|
}
|
|
844
832
|
// Otherwise, i.e. if it is not authenticated:
|
|
845
833
|
else {
|
|
@@ -903,11 +891,6 @@ const requestHandler = async (request, response) => {
|
|
|
903
891
|
response.end(JSON.stringify({status: 'error', message: answerData.message}));
|
|
904
892
|
}
|
|
905
893
|
}
|
|
906
|
-
// Otherwise, i.e. if it is any other POST request:
|
|
907
|
-
else {
|
|
908
|
-
// Report its invalidity.
|
|
909
|
-
await serveError({message: 'ERROR: Invalid POST request'}, response, true);
|
|
910
|
-
}
|
|
911
894
|
}
|
|
912
895
|
}
|
|
913
896
|
// Otherwise, i.e. if it is neither a GET nor a POST request:
|
|
@@ -928,36 +911,40 @@ const serve = async (protocolModule, options) => {
|
|
|
928
911
|
for (const path of [queuePath(), claimedPath(), failedPath(), hiddenReportsPath(), reportsPath()]) {
|
|
929
912
|
await fs.mkdir(path, {recursive: true});
|
|
930
913
|
}
|
|
931
|
-
const server = protocolModule ===
|
|
914
|
+
const server = protocolModule === https
|
|
932
915
|
? https.createServer(options, requestHandler)
|
|
933
916
|
: http.createServer(requestHandler);
|
|
934
917
|
const port = process.env.PORT || '3000';
|
|
935
918
|
server.listen(port, () => {
|
|
936
919
|
console.log(`Kilotest server listening at ${protocol}://localhost:${port}.`);
|
|
937
920
|
});
|
|
921
|
+
return server;
|
|
938
922
|
};
|
|
939
923
|
|
|
940
|
-
|
|
924
|
+
exports.serve = serve;
|
|
941
925
|
|
|
942
|
-
|
|
943
|
-
|
|
926
|
+
// Starts the server using the configured protocol and credentials.
|
|
927
|
+
exports.startServer = async () => {
|
|
928
|
+
const startProtocol = process.env.PROTOCOL || 'http';
|
|
929
|
+
if (startProtocol === 'http') {
|
|
944
930
|
console.log('Starting HTTP server');
|
|
945
|
-
serve(http, {});
|
|
931
|
+
return serve(http, {});
|
|
946
932
|
}
|
|
947
|
-
else if (
|
|
933
|
+
else if (startProtocol === 'https') {
|
|
948
934
|
console.log('Starting HTTPS server');
|
|
949
|
-
fs.readFile(process.env.KEY, 'utf8')
|
|
950
|
-
.
|
|
951
|
-
|
|
952
|
-
fs.readFile(process.env.CERT, 'utf8')
|
|
953
|
-
.then(
|
|
954
|
-
cert => {
|
|
955
|
-
serve(https, {key, cert});
|
|
956
|
-
},
|
|
957
|
-
error => console.log(error.message)
|
|
958
|
-
);
|
|
959
|
-
},
|
|
960
|
-
error => console.log(error.message)
|
|
961
|
-
);
|
|
935
|
+
const key = await fs.readFile(process.env.KEY, 'utf8');
|
|
936
|
+
const cert = await fs.readFile(process.env.CERT, 'utf8');
|
|
937
|
+
return serve(https, {key, cert});
|
|
962
938
|
}
|
|
963
|
-
}
|
|
939
|
+
};
|
|
940
|
+
|
|
941
|
+
// Runs the server if the module was loaded directly (not required by a test).
|
|
942
|
+
exports.runIfMain = (mainModule, currentModule) => {
|
|
943
|
+
if (mainModule === currentModule) {
|
|
944
|
+
exports.startServer().catch(error => console.log(error.message));
|
|
945
|
+
}
|
|
946
|
+
};
|
|
947
|
+
|
|
948
|
+
// EXECUTION
|
|
949
|
+
|
|
950
|
+
exports.runIfMain(require.main, module);
|