@jrpool/kilotest 47.0.1 → 49.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/scheduled_tasks.lock +1 -0
- package/api/requestFeature.test.ts +4 -2
- package/api/requestFeature.ts +1 -1
- package/api/requestRetest.test.ts +66 -7
- package/api/requestRetest.ts +55 -39
- package/api/requestTest.test.ts +63 -7
- package/api/requestTest.ts +36 -26
- package/api/util.test.ts +6 -5
- package/api/util.ts +2 -24
- package/docs/BACKLOG.md +228 -16
- package/docs/SERVICE.md +84 -7
- package/index.test.ts +354 -25
- package/index.ts +174 -8
- package/linkValidation.test.ts +1 -0
- package/mcp.test.ts +30 -0
- package/mcp.ts +9 -0
- package/package.json +1 -1
- package/smokeTest.ts +16 -0
- package/test/dbFixture.ts +8 -0
- package/util.test.ts +426 -127
- package/util.ts +389 -196
- package/web/ai0BalanceForm/ai0BalanceForm.test.ts +16 -10
- package/web/ai0BalanceForm/index.html +1 -1
- package/web/ai0BalanceForm/index.ts +10 -6
- package/web/enqueue/index.ts +5 -2
- package/web/expungeReportsForm/expungeReportsForm.test.ts +19 -11
- package/web/expungeReportsForm/index.html +1 -1
- package/web/expungeReportsForm/index.ts +1 -1
- package/web/hideReportForm/hideReportForm.test.ts +24 -9
- package/web/hideReportForm/index.html +1 -1
- package/web/hideReportForm/index.ts +9 -6
- package/web/listReports/index.ts +33 -12
- package/web/listReports/listReports.test.ts +1 -1
- package/web/listTopIssues/index.ts +1 -1
- package/web/manage/index.html +2 -1
- package/web/metrics/index.html +19 -0
- package/web/metrics/index.ts +126 -0
- package/web/metrics/metrics.test.ts +211 -0
- package/web/pruneReportsForm/index.html +1 -1
- package/web/pruneReportsForm/index.ts +1 -1
- package/web/pruneReportsForm/pruneReportsForm.test.ts +18 -10
- package/web/reannotate/index.ts +7 -3
- package/web/reannotate/reannotate.test.ts +1 -1
- package/web/renewWCAG/index.ts +4 -1
- package/web/renewWCAG/renewWCAG.test.ts +1 -1
- package/web/reportDeletion.ts +12 -7
- package/web/requestRetest/index.html +3 -3
- package/web/requestRetest/index.ts +47 -16
- package/web/requestRetest/requestRetest.test.ts +38 -24
- package/web/requestTest/index.html +4 -4
- package/web/requestTest/index.ts +37 -9
- package/web/requestTest/requestTest.test.ts +21 -6
- package/web/rewindReportsForm/index.html +1 -1
- package/web/rewindReportsForm/index.ts +1 -1
- package/web/rewindReportsForm/rewindReportsForm.test.ts +18 -10
- package/web/showHiddenReportsForm/index.html +24 -0
- package/web/showHiddenReportsForm/index.ts +45 -0
- package/web/showHiddenReportsForm/showHiddenReportsForm.test.ts +103 -0
- package/web/tutorialAI/index.ts +1 -1
- package/web/tutorialWeb/index.ts +1 -1
- package/web/unhideReportForm/index.html +1 -1
- package/web/unhideReportForm/index.ts +11 -8
- package/web/unhideReportForm/unhideReportForm.test.ts +19 -11
|
@@ -44,22 +44,28 @@ after(async () => {
|
|
|
44
44
|
|
|
45
45
|
// TESTS
|
|
46
46
|
|
|
47
|
-
test('ai0BalanceForm displays the current balance
|
|
48
|
-
const result: any = await answer(null, '');
|
|
47
|
+
test('ai0BalanceForm displays the current balance on a GET request', async () => {
|
|
48
|
+
const result: any = await answer(null, '', 'GET');
|
|
49
49
|
assert.equal(result.status, 'ok');
|
|
50
50
|
assert.ok(result.answerPage);
|
|
51
51
|
const html = parse(result.answerPage);
|
|
52
52
|
assert.ok(html.querySelector('title'));
|
|
53
53
|
});
|
|
54
54
|
|
|
55
|
-
test('ai0BalanceForm
|
|
56
|
-
const result: any = await answer(null, 'authCode=
|
|
55
|
+
test('ai0BalanceForm ignores a newBalance query string on a GET request', async () => {
|
|
56
|
+
const result: any = await answer(null, 'authCode=test-auth-code&newBalance=99.00', 'GET');
|
|
57
|
+
assert.equal(result.status, 'ok');
|
|
58
|
+
assert.ok(!result.answerPage.includes('$99 is the'));
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
test('ai0BalanceForm returns an error for an invalid auth code on a POST request', async () => {
|
|
62
|
+
const result: any = await answer(null, 'authCode=wrong&newBalance=5.00', 'POST');
|
|
57
63
|
assert.equal(result.status, 'error');
|
|
58
|
-
assert.equal(result.message, 'Invalid
|
|
64
|
+
assert.equal(result.message, 'Invalid request');
|
|
59
65
|
});
|
|
60
66
|
|
|
61
|
-
test('ai0BalanceForm records a valid new balance with valid auth code', async () => {
|
|
62
|
-
const result: any = await answer(null, 'authCode=test-auth-code&newBalance=2.50');
|
|
67
|
+
test('ai0BalanceForm records a valid new balance with valid auth code on a POST request', async () => {
|
|
68
|
+
const result: any = await answer(null, 'authCode=test-auth-code&newBalance=2.50', 'POST');
|
|
63
69
|
assert.equal(result.status, 'ok');
|
|
64
70
|
assert.ok(result.answerPage.includes('$2.5 is the'));
|
|
65
71
|
// Verify the file was written.
|
|
@@ -67,9 +73,9 @@ test('ai0BalanceForm records a valid new balance with valid auth code', async ()
|
|
|
67
73
|
assert.equal(data.balance, 2.50);
|
|
68
74
|
});
|
|
69
75
|
|
|
70
|
-
test('ai0BalanceForm does not record an invalid balance', async () => {
|
|
76
|
+
test('ai0BalanceForm does not record an invalid balance on a POST request', async () => {
|
|
71
77
|
// 150 is out of range (>= 100).
|
|
72
|
-
const result: any = await answer(null, 'authCode=test-auth-code&newBalance=150');
|
|
78
|
+
const result: any = await answer(null, 'authCode=test-auth-code&newBalance=150', 'POST');
|
|
73
79
|
assert.equal(result.status, 'ok');
|
|
74
80
|
// The oldBalance should not show the new value.
|
|
75
81
|
assert.ok(!result.answerPage.includes('$150 is the'));
|
|
@@ -78,7 +84,7 @@ test('ai0BalanceForm does not record an invalid balance', async () => {
|
|
|
78
84
|
test('ai0BalanceForm shows no-balance message when balance file is missing', async () => {
|
|
79
85
|
await fs.unlink(balancePath).catch(() => {});
|
|
80
86
|
try {
|
|
81
|
-
const result: any = await answer(null, '');
|
|
87
|
+
const result: any = await answer(null, '', 'GET');
|
|
82
88
|
assert.equal(result.status, 'ok');
|
|
83
89
|
assert.ok(result.answerPage.includes('There is no'));
|
|
84
90
|
}
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
<h1><a href="/">Kilotest</a>: Record AI service 0 balance</h1>
|
|
16
16
|
<p>__oldBalance__ recorded AI service 0 balance.</p>
|
|
17
17
|
<p>The current AI service 0 balance is available on the <a href="https://platform.claude.com/settings/billing">Claude Platform Billing page</a>.</p>
|
|
18
|
-
<form action="/ai0BalanceForm.html">
|
|
18
|
+
<form action="/ai0BalanceForm.html" method="post">
|
|
19
19
|
<p><label>
|
|
20
20
|
Balance (in $):
|
|
21
21
|
<input type="number" size="5" min="0" max="99.99" step="0.01" name="newBalance" required>
|
|
@@ -11,14 +11,16 @@ import {getJSON, populateTemplate} from '../../util.ts';
|
|
|
11
11
|
|
|
12
12
|
// FUNCTIONS
|
|
13
13
|
|
|
14
|
-
// Returns a form for recording the AI service 0 balance.
|
|
15
|
-
|
|
14
|
+
// Returns a form for recording the AI service 0 balance. A GET request never processes a
|
|
15
|
+
// submission, regardless of its query string; only a POST request (the form's own
|
|
16
|
+
// submission) does.
|
|
17
|
+
export const answer = async (_: any, search: string, method: string) => {
|
|
16
18
|
const searchParams = new URLSearchParams(search);
|
|
17
19
|
const authCode = searchParams?.get('authCode');
|
|
18
20
|
const newBalanceString = searchParams?.get('newBalance');
|
|
19
21
|
let oldBalance: any;
|
|
20
|
-
// If the form
|
|
21
|
-
if (newBalanceString) {
|
|
22
|
+
// If the form has been submitted:
|
|
23
|
+
if (method === 'POST' && newBalanceString) {
|
|
22
24
|
// If the authorization code is valid:
|
|
23
25
|
if (authCode === process.env.AUTH_CODE) {
|
|
24
26
|
const newBalance = Number.parseFloat(newBalanceString);
|
|
@@ -40,10 +42,12 @@ export const answer = async (_: any, search: string) => {
|
|
|
40
42
|
}
|
|
41
43
|
// Otherwise, i.e. if the authorization code is invalid:
|
|
42
44
|
else {
|
|
43
|
-
// Report the error
|
|
45
|
+
// Report the error, deliberately vague so as not to confirm to an attacker that the
|
|
46
|
+
// authorization code specifically (as opposed to some other part of the request) is
|
|
47
|
+
// what was wrong.
|
|
44
48
|
return {
|
|
45
49
|
status: 'error',
|
|
46
|
-
message: 'Invalid
|
|
50
|
+
message: 'Invalid request'
|
|
47
51
|
}
|
|
48
52
|
}
|
|
49
53
|
}
|
package/web/enqueue/index.ts
CHANGED
|
@@ -48,9 +48,12 @@ export const answer = async (url: string, description: string, authCode: string)
|
|
|
48
48
|
answerPage
|
|
49
49
|
};
|
|
50
50
|
}
|
|
51
|
-
// Otherwise, i.e. if the authorization code is invalid, return an error page.
|
|
51
|
+
// Otherwise, i.e. if the authorization code is invalid, return an error page. The
|
|
52
|
+
// message is deliberately vague so as not to confirm to an attacker that the
|
|
53
|
+
// authorization code specifically (as opposed to some other part of the request) is
|
|
54
|
+
// what was wrong.
|
|
52
55
|
return {
|
|
53
56
|
status: 'error',
|
|
54
|
-
message: 'Invalid
|
|
57
|
+
message: 'Invalid request'
|
|
55
58
|
};
|
|
56
59
|
};
|
|
@@ -35,32 +35,40 @@ after(async () => {
|
|
|
35
35
|
|
|
36
36
|
// TESTS
|
|
37
37
|
|
|
38
|
-
test('expungeReportsForm displays a list of reports
|
|
39
|
-
const result: any = await answer(null, '');
|
|
38
|
+
test('expungeReportsForm displays a list of reports on a GET request', async () => {
|
|
39
|
+
const result: any = await answer(null, '', 'GET');
|
|
40
40
|
assert.equal(result.status, 'ok');
|
|
41
41
|
assert.ok(result.answerPage);
|
|
42
42
|
const html = parse(result.answerPage);
|
|
43
43
|
assert.ok(html.querySelector('title'));
|
|
44
44
|
});
|
|
45
45
|
|
|
46
|
-
test('expungeReportsForm
|
|
47
|
-
const
|
|
46
|
+
test('expungeReportsForm ignores a report query string on a GET request', async () => {
|
|
47
|
+
const reportPath = path.join(reportsPath(), '260101T0001-ct.json');
|
|
48
|
+
const result: any = await answer(null, 'authCode=test-auth-code&report=260101T0001-ct', 'GET');
|
|
49
|
+
assert.equal(result.status, 'ok');
|
|
50
|
+
// The report should NOT have been deleted by a mere GET request.
|
|
51
|
+
await fs.access(reportPath);
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
test('expungeReportsForm returns an error for an invalid auth code on a POST request', async () => {
|
|
55
|
+
const result: any = await answer(null, 'authCode=wrong&report=260101T0001-ct', 'POST');
|
|
48
56
|
assert.equal(result.status, 'error');
|
|
49
|
-
assert.equal(result.message, 'Invalid
|
|
57
|
+
assert.equal(result.message, 'Invalid request');
|
|
50
58
|
});
|
|
51
59
|
|
|
52
|
-
test('expungeReportsForm returns an error when deleting a nonexistent report', async () => {
|
|
53
|
-
const result: any = await answer(null, 'authCode=test-auth-code&report=999999T9999-nope');
|
|
60
|
+
test('expungeReportsForm returns an error when deleting a nonexistent report on a POST request', async () => {
|
|
61
|
+
const result: any = await answer(null, 'authCode=test-auth-code&report=999999T9999-nope', 'POST');
|
|
54
62
|
assert.equal(result.status, 'error');
|
|
55
63
|
assert.ok(result.message.includes('Deleting sole reports'));
|
|
56
64
|
});
|
|
57
65
|
|
|
58
|
-
test('expungeReportsForm deletes a sole report with valid auth code', async () => {
|
|
66
|
+
test('expungeReportsForm deletes a sole report with valid auth code on a POST request', async () => {
|
|
59
67
|
// 260101T0001-ct is the only report for its URL (canttell).
|
|
60
68
|
const reportPath = path.join(reportsPath(), '260101T0001-ct.json');
|
|
61
69
|
const backup = await fs.readFile(reportPath, 'utf8');
|
|
62
70
|
try {
|
|
63
|
-
const result: any = await answer(null, 'authCode=test-auth-code&report=260101T0001-ct');
|
|
71
|
+
const result: any = await answer(null, 'authCode=test-auth-code&report=260101T0001-ct', 'POST');
|
|
64
72
|
assert.equal(result.status, 'ok');
|
|
65
73
|
await fs.access(reportPath).then(
|
|
66
74
|
() => {throw new Error('Report should have been deleted');},
|
|
@@ -77,7 +85,7 @@ test('expungeReportsForm returns an error when a report file is corrupt', async
|
|
|
77
85
|
const backup = await fs.readFile(reportPath, 'utf8');
|
|
78
86
|
try {
|
|
79
87
|
await fs.writeFile(reportPath, 'not valid json');
|
|
80
|
-
const result: any = await answer(null, '');
|
|
88
|
+
const result: any = await answer(null, '', 'GET');
|
|
81
89
|
assert.equal(result.status, 'error');
|
|
82
90
|
}
|
|
83
91
|
finally {
|
|
@@ -100,7 +108,7 @@ test('expungeReportsForm shows no-deletable message when every URL has at least
|
|
|
100
108
|
process.env.DB_DIR = tmpDir;
|
|
101
109
|
try {
|
|
102
110
|
const {answer} = await import('./index.ts');
|
|
103
|
-
const result: any = await answer(null, '');
|
|
111
|
+
const result: any = await answer(null, '', 'GET');
|
|
104
112
|
assert.equal(result.status, 'ok');
|
|
105
113
|
assert.ok(result.answerPage.includes('no reports to delete'));
|
|
106
114
|
assert.ok(result.answerPage.includes('disabled'));
|
|
@@ -10,7 +10,7 @@ import {reportDeletionForm} from '../reportDeletion.ts';
|
|
|
10
10
|
// FUNCTIONS
|
|
11
11
|
|
|
12
12
|
// Returns a form for deleting sole reports.
|
|
13
|
-
export const answer = async (_: any, search: string) => reportDeletionForm(import.meta.dirname, search, {
|
|
13
|
+
export const answer = async (_: any, search: string, method: string) => reportDeletionForm(import.meta.dirname, search, method, {
|
|
14
14
|
intro: 'Choose the sole reports to delete.',
|
|
15
15
|
emptyIntro: 'Each target has at least 2 reports, so there are no reports to delete.',
|
|
16
16
|
failurePrefix: 'Deleting sole reports failed',
|
|
@@ -44,8 +44,8 @@ after(async () => {
|
|
|
44
44
|
|
|
45
45
|
// TESTS
|
|
46
46
|
|
|
47
|
-
test('hideReportForm displays a list of reports
|
|
48
|
-
const result: any = await answer(null, '');
|
|
47
|
+
test('hideReportForm displays a list of reports on a GET request', async () => {
|
|
48
|
+
const result: any = await answer(null, '', 'GET');
|
|
49
49
|
assert.equal(result.status, 'ok');
|
|
50
50
|
assert.ok(result.answerPage);
|
|
51
51
|
const html = parse(result.answerPage);
|
|
@@ -53,13 +53,28 @@ test('hideReportForm displays a list of reports when no submission', async () =>
|
|
|
53
53
|
assert.ok(radios.length > 0);
|
|
54
54
|
});
|
|
55
55
|
|
|
56
|
-
test('hideReportForm
|
|
57
|
-
const
|
|
56
|
+
test('hideReportForm ignores a report query string on a GET request', async () => {
|
|
57
|
+
const reportPath = path.join(reportsPath(), '260101T0009-brd.json');
|
|
58
|
+
const hiddenPath = path.join(hiddenReportsPath(), '260101T0009-brd.json');
|
|
59
|
+
try {
|
|
60
|
+
await fs.rename(hiddenPath, reportPath);
|
|
61
|
+
}
|
|
62
|
+
catch {
|
|
63
|
+
// Report may already be in reports.
|
|
64
|
+
}
|
|
65
|
+
const result: any = await answer(null, 'authCode=test-auth-code&report=260101T0009-brd', 'GET');
|
|
66
|
+
assert.equal(result.status, 'ok');
|
|
67
|
+
// The report should NOT have been hidden by a mere GET request.
|
|
68
|
+
await fs.access(reportPath);
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
test('hideReportForm returns an error for an invalid auth code on a POST request', async () => {
|
|
72
|
+
const result: any = await answer(null, 'authCode=wrong&report=260101T0009-brd', 'POST');
|
|
58
73
|
assert.equal(result.status, 'error');
|
|
59
|
-
assert.equal(result.message, 'Invalid
|
|
74
|
+
assert.equal(result.message, 'Invalid request');
|
|
60
75
|
});
|
|
61
76
|
|
|
62
|
-
test('hideReportForm hides a report with valid auth code', async () => {
|
|
77
|
+
test('hideReportForm hides a report with valid auth code on a POST request', async () => {
|
|
63
78
|
const reportPath = path.join(reportsPath(), '260101T0009-brd.json');
|
|
64
79
|
const hiddenPath = path.join(hiddenReportsPath(), '260101T0009-brd.json');
|
|
65
80
|
// Ensure the report exists in reports and not in hidden.
|
|
@@ -69,14 +84,14 @@ test('hideReportForm hides a report with valid auth code', async () => {
|
|
|
69
84
|
catch {
|
|
70
85
|
// Report may already be in reports.
|
|
71
86
|
}
|
|
72
|
-
const result: any = await answer(null, 'authCode=test-auth-code&report=260101T0009-brd');
|
|
87
|
+
const result: any = await answer(null, 'authCode=test-auth-code&report=260101T0009-brd', 'POST');
|
|
73
88
|
assert.equal(result.status, 'ok');
|
|
74
89
|
// The report should now be in the hidden directory.
|
|
75
90
|
await fs.access(hiddenPath);
|
|
76
91
|
});
|
|
77
92
|
|
|
78
|
-
test('hideReportForm returns an error when hiding a nonexistent report', async () => {
|
|
79
|
-
const result: any = await answer(null, 'authCode=test-auth-code&report=999999T9999-nope');
|
|
93
|
+
test('hideReportForm returns an error when hiding a nonexistent report on a POST request', async () => {
|
|
94
|
+
const result: any = await answer(null, 'authCode=test-auth-code&report=999999T9999-nope', 'POST');
|
|
80
95
|
assert.equal(result.status, 'error');
|
|
81
96
|
assert.ok(result.message.includes('Hiding report'));
|
|
82
97
|
});
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
<main>
|
|
15
15
|
<h1><a href="/">Kilotest</a>: Classify report as experimental</h1>
|
|
16
16
|
<p>Choose a report to classify as experimental.</p>
|
|
17
|
-
<form action="/hideReportForm.html">
|
|
17
|
+
<form action="/hideReportForm.html" method="post">
|
|
18
18
|
<fieldset>
|
|
19
19
|
<legend>Reports</legend>
|
|
20
20
|
__reports__
|
|
@@ -11,13 +11,14 @@ import path from 'node:path';
|
|
|
11
11
|
|
|
12
12
|
// FUNCTIONS
|
|
13
13
|
|
|
14
|
-
// Returns a form for hiding a report.
|
|
15
|
-
|
|
14
|
+
// Returns a form for hiding a report. A GET request never processes a submission,
|
|
15
|
+
// regardless of its query string; only a POST request (the form's own submission) does.
|
|
16
|
+
export const answer = async (_: any, search: string, method: string) => {
|
|
16
17
|
const searchParams = new URLSearchParams(search);
|
|
17
18
|
const authCode = searchParams?.get('authCode');
|
|
18
19
|
const jobName = searchParams?.get('report');
|
|
19
|
-
// If the form has been
|
|
20
|
-
if (jobName) {
|
|
20
|
+
// If the form has been submitted and a report is to be hidden:
|
|
21
|
+
if (method === 'POST' && jobName) {
|
|
21
22
|
// If the authorization code is valid:
|
|
22
23
|
if (authCode === process.env.AUTH_CODE) {
|
|
23
24
|
const fileName = `${jobName}.json`;
|
|
@@ -36,10 +37,12 @@ export const answer = async (_: any, search: string) => {
|
|
|
36
37
|
}
|
|
37
38
|
// Otherwise, i.e. if the authorization code is invalid:
|
|
38
39
|
else {
|
|
39
|
-
// Report the error
|
|
40
|
+
// Report the error, deliberately vague so as not to confirm to an attacker that the
|
|
41
|
+
// authorization code specifically (as opposed to some other part of the request) is
|
|
42
|
+
// what was wrong.
|
|
40
43
|
return {
|
|
41
44
|
status: 'error',
|
|
42
|
-
message: 'Invalid
|
|
45
|
+
message: 'Invalid request'
|
|
43
46
|
}
|
|
44
47
|
}
|
|
45
48
|
}
|
package/web/listReports/index.ts
CHANGED
|
@@ -15,7 +15,6 @@ import {
|
|
|
15
15
|
getTestRequests,
|
|
16
16
|
getReportData,
|
|
17
17
|
getReportExtracts,
|
|
18
|
-
getRequestability,
|
|
19
18
|
jobsPath,
|
|
20
19
|
objectSort,
|
|
21
20
|
populateTemplate
|
|
@@ -35,14 +34,14 @@ const populateQuery = async (query: Record<string, any>) => {
|
|
|
35
34
|
tested: []
|
|
36
35
|
};
|
|
37
36
|
// Get the test requests.
|
|
38
|
-
const testRequests = await getTestRequests()
|
|
37
|
+
const testRequests = await getTestRequests();
|
|
39
38
|
// For each requested URL:
|
|
40
39
|
Object.keys(testRequests).forEach(url => {
|
|
41
40
|
// For each of its requests:
|
|
42
41
|
testRequests[url]!.forEach((req: any) => {
|
|
43
|
-
const {description,
|
|
42
|
+
const {description, reason} = req;
|
|
44
43
|
// Add a line.
|
|
45
|
-
lines.requests.push(`${margin}<li><code>${url}</code> (${description}): ${
|
|
44
|
+
lines.requests.push(`${margin}<li><code>${url}</code> (${description}): ${reason}</li>`);
|
|
46
45
|
});
|
|
47
46
|
});
|
|
48
47
|
// Sort the lines in alphabetical order by URL and secondarily by proposed name.
|
|
@@ -55,12 +54,29 @@ const populateQuery = async (query: Record<string, any>) => {
|
|
|
55
54
|
: 'No requests await approval now.';
|
|
56
55
|
// Get the file names of all queued and claimed jobs.
|
|
57
56
|
const jobFileNames = await getJobNames();
|
|
57
|
+
// Initialize sets of the page descriptions and URLs of jobs in both categories.
|
|
58
|
+
const jobsData = {
|
|
59
|
+
queue: {
|
|
60
|
+
descriptions: new Set(),
|
|
61
|
+
urls: new Set()
|
|
62
|
+
},
|
|
63
|
+
claimed: {
|
|
64
|
+
descriptions: new Set(),
|
|
65
|
+
urls: new Set()
|
|
66
|
+
}
|
|
67
|
+
};
|
|
58
68
|
// For each job category:
|
|
59
69
|
for (const category of ['queue', 'claimed'] as const) {
|
|
60
70
|
// For each job in the category:
|
|
61
71
|
for (const fileName of jobFileNames[category]) {
|
|
62
72
|
// Get the job.
|
|
63
73
|
const job = await getObject(path.join(jobsPath(), category, fileName)) as {target: {url: string, what: string}};
|
|
74
|
+
// Get the description and URL of its page.
|
|
75
|
+
const {target} = job;
|
|
76
|
+
const {what, url} = target;
|
|
77
|
+
// Ensure they are in the sets of properties of the category.
|
|
78
|
+
jobsData[category].descriptions.add(what);
|
|
79
|
+
jobsData[category].urls.add(url);
|
|
64
80
|
// Add a line.
|
|
65
81
|
lines[category].push(`${margin}<li><code>${job.target.url}</code> (${job.target.what})</li>`);
|
|
66
82
|
}
|
|
@@ -84,7 +100,7 @@ const populateQuery = async (query: Record<string, any>) => {
|
|
|
84
100
|
sortedExtracts = objectSort(sortedExtracts, 'description', 'alpha');
|
|
85
101
|
// For each report:
|
|
86
102
|
for (const extract of sortedExtracts) {
|
|
87
|
-
const {jobID, timeStamp, url, description} = extract;
|
|
103
|
+
const {jobID, timeStamp, url, description, superseded} = extract;
|
|
88
104
|
// Get data about it.
|
|
89
105
|
const reportData = await getReportData(timeStamp, jobID);
|
|
90
106
|
// If this failed:
|
|
@@ -145,21 +161,26 @@ const populateQuery = async (query: Record<string, any>) => {
|
|
|
145
161
|
const link = `<a ${href} ${label}>What ${questionString}?</a>`;
|
|
146
162
|
lines.tested.push(`${margin} <li>${link}</li>`);
|
|
147
163
|
}
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
164
|
+
let retestString: string = '';
|
|
165
|
+
// If a page with the same description or URL is being tested:
|
|
166
|
+
if (jobsData.claimed.descriptions.has(description) || jobsData.claimed.urls.has(url)) {
|
|
167
|
+
// Report this.
|
|
152
168
|
retestString = 'Currently being retested';
|
|
153
169
|
}
|
|
154
|
-
|
|
170
|
+
// Otherwise, if such a page is queued:
|
|
171
|
+
else if (jobsData.queue.descriptions.has(description) || jobsData.queue.urls.has(url)) {
|
|
155
172
|
retestString = 'Currently in the queue for retesting';
|
|
156
173
|
}
|
|
157
|
-
|
|
174
|
+
// Otherwise, if no such page is being tested or queued and the report is not superseded:
|
|
175
|
+
else if (!superseded) {
|
|
176
|
+
// Make the page available for retesting.
|
|
158
177
|
const href = `/requestRetestForm.html/${timeStamp}/${jobID}`;
|
|
159
178
|
const retestContent = 'Should Kilotest retest the page?';
|
|
160
179
|
retestString = `<a href="${href}">${retestContent}</a>`;
|
|
161
180
|
}
|
|
162
|
-
|
|
181
|
+
if (retestString) {
|
|
182
|
+
lines.tested.push(`${margin} <li>${retestString}</li>`);
|
|
183
|
+
}
|
|
163
184
|
lines.tested.push(`${margin} </ul>`);
|
|
164
185
|
lines.tested.push(`${margin}</details>`);
|
|
165
186
|
}
|
|
@@ -95,7 +95,7 @@ test('listReports shows requests when testRequests.json has entries', {timeout:
|
|
|
95
95
|
try {
|
|
96
96
|
const testRequests = {
|
|
97
97
|
'https://example.com/mixed': [
|
|
98
|
-
{description: 'Mixed Outcomes Page',
|
|
98
|
+
{description: 'Mixed Outcomes Page', reason: 'Needs retesting for accessibility'}
|
|
99
99
|
]
|
|
100
100
|
};
|
|
101
101
|
await fs.writeFile(testRequestsPath, JSON.stringify(testRequests, null, 2));
|
|
@@ -79,7 +79,7 @@ const getIssuesSummary = async () => {
|
|
|
79
79
|
// Report this.
|
|
80
80
|
console.log(`ERROR: Annotations obsolete for issue ${issueID}; reannotate`);
|
|
81
81
|
// Notify a manager.
|
|
82
|
-
sendAlert('Annotations obsolete', `Annotations for issue ${issueID} obsolete; reannotate.`);
|
|
82
|
+
sendAlert('Kilotest: Annotations obsolete', `Annotations for issue ${issueID} obsolete; reannotate.`);
|
|
83
83
|
}
|
|
84
84
|
});
|
|
85
85
|
// For each summarized issue:
|
package/web/manage/index.html
CHANGED
|
@@ -23,9 +23,10 @@
|
|
|
23
23
|
<li><a href="/rewindReportsForm.html">Delete latest superseding reports</a></li>
|
|
24
24
|
<li><a href="/expungeReportsForm.html">Delete sole reports</a></li>
|
|
25
25
|
<li><a href="/hideReportForm.html">Classify a report as experimental</a></li>
|
|
26
|
-
<li><a href="/
|
|
26
|
+
<li><a href="/showHiddenReportsForm.html">View the list of experimental reports</a></li>
|
|
27
27
|
<li><a href="/ai0BalanceForm.html">Record the current balance of AI service 0</a></li>
|
|
28
28
|
<li><a href="/renewWCAGForm.html">Renew the map of WCAG numeric to text identifiers</a></li>
|
|
29
|
+
<li><a href="/metrics.html">View usage metrics</a></li>
|
|
29
30
|
</ul>
|
|
30
31
|
</main>
|
|
31
32
|
</body>
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="en-US">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="utf-8">
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
6
|
+
<meta name="publisher" content="Jonathan Robert Pool">
|
|
7
|
+
<meta name="creator" content="Jonathan Robert Pool">
|
|
8
|
+
<meta name="keywords" content="report,accessibility,a11y">
|
|
9
|
+
<title>Usage metrics | Kilotest</title>
|
|
10
|
+
<link rel="icon" href="/favicon.ico">
|
|
11
|
+
<link rel="stylesheet" href="/style.css">
|
|
12
|
+
</head>
|
|
13
|
+
<body>
|
|
14
|
+
<main>
|
|
15
|
+
<h1><a href="/">Kilotest</a>: Usage metrics</h1>
|
|
16
|
+
__body__
|
|
17
|
+
</main>
|
|
18
|
+
</body>
|
|
19
|
+
</html>
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
/*
|
|
2
|
+
index.ts
|
|
3
|
+
Serves usage metrics for the maintainer: web page views, MCP tool calls, and API
|
|
4
|
+
operation calls, each broken down by name, recorded since a given date and time.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
// IMPORTS
|
|
8
|
+
|
|
9
|
+
import {
|
|
10
|
+
clearMetrics, getDateTimeString, getExclusionCookieValue, getMetrics, metricsExclusionCookieName, populateTemplate
|
|
11
|
+
} from '../../util.ts';
|
|
12
|
+
|
|
13
|
+
// FUNCTIONS
|
|
14
|
+
|
|
15
|
+
// Returns an HTML table of the counts in a metrics category, sorted by descending count.
|
|
16
|
+
const getCategoryTable = (counts: Record<string, number>): string => {
|
|
17
|
+
const margin = ' '.repeat(6);
|
|
18
|
+
const names = Object.keys(counts).sort((a, b) => counts[b]! - counts[a]!);
|
|
19
|
+
if (!names.length) {
|
|
20
|
+
return `${margin}<p>None yet.</p>`;
|
|
21
|
+
}
|
|
22
|
+
const rows = names.map(name => `${margin} <tr><td>${name}</td><td>${counts[name]}</td></tr>`);
|
|
23
|
+
return [
|
|
24
|
+
`${margin}<table class="allBorder">`,
|
|
25
|
+
`${margin} <thead>`,
|
|
26
|
+
`${margin} <tr><th>Name</th><th>Count</th></tr>`,
|
|
27
|
+
`${margin} </thead>`,
|
|
28
|
+
`${margin} <tbody>`,
|
|
29
|
+
...rows,
|
|
30
|
+
`${margin} </tbody>`,
|
|
31
|
+
`${margin}</table>`
|
|
32
|
+
].join('\n');
|
|
33
|
+
};
|
|
34
|
+
// Returns an HTML table of manager-page activity, sorted by descending failure count,
|
|
35
|
+
// since repeated failures are the signal of suspected abuse most worth surfacing first.
|
|
36
|
+
const getManagerActivityTable = (activity: Record<string, {ok: number; error: number}>): string => {
|
|
37
|
+
const margin = ' '.repeat(6);
|
|
38
|
+
const names = Object.keys(activity).sort((a, b) => activity[b]!.error - activity[a]!.error);
|
|
39
|
+
if (!names.length) {
|
|
40
|
+
return `${margin}<p>None yet.</p>`;
|
|
41
|
+
}
|
|
42
|
+
const rows = names.map(name => {
|
|
43
|
+
const {ok, error} = activity[name]!;
|
|
44
|
+
return `${margin} <tr><td>${name}</td><td>${ok}</td><td>${error}</td></tr>`;
|
|
45
|
+
});
|
|
46
|
+
return [
|
|
47
|
+
`${margin}<table class="allBorder">`,
|
|
48
|
+
`${margin} <thead>`,
|
|
49
|
+
`${margin} <tr><th>Name</th><th>Successful</th><th>Failed</th></tr>`,
|
|
50
|
+
`${margin} </thead>`,
|
|
51
|
+
`${margin} <tbody>`,
|
|
52
|
+
...rows,
|
|
53
|
+
`${margin} </tbody>`,
|
|
54
|
+
`${margin}</table>`
|
|
55
|
+
].join('\n');
|
|
56
|
+
};
|
|
57
|
+
// Returns a page reporting usage metrics, or, if the form has not been submitted (a POST
|
|
58
|
+
// request) yet, a form requesting an authorization code. A GET request, regardless of its
|
|
59
|
+
// query string, never processes a submission, so the action cannot be triggered by GET.
|
|
60
|
+
export const answer = async (_: any, search: string, method: string) => {
|
|
61
|
+
const searchParams = new URLSearchParams(search);
|
|
62
|
+
const authCode = searchParams?.get('authCode');
|
|
63
|
+
let query: Record<string, string>;
|
|
64
|
+
// If the form has been submitted:
|
|
65
|
+
if (method === 'POST') {
|
|
66
|
+
// If the authorization code is invalid:
|
|
67
|
+
if (authCode !== process.env.AUTH_CODE) {
|
|
68
|
+
// Report the error, deliberately vague so as not to confirm to an attacker that the
|
|
69
|
+
// authorization code specifically (as opposed to some other part of the request) is
|
|
70
|
+
// what was wrong.
|
|
71
|
+
return {
|
|
72
|
+
status: 'error',
|
|
73
|
+
message: 'Invalid request'
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
// If clearing the counts was requested:
|
|
77
|
+
const clearCounts = searchParams.has('clearCounts');
|
|
78
|
+
const metrics = clearCounts ? await clearMetrics() : await getMetrics();
|
|
79
|
+
query = {
|
|
80
|
+
body: [
|
|
81
|
+
...(clearCounts ? ['<p>Counts cleared.</p>'] : []),
|
|
82
|
+
`<p>Counts recorded since ${getDateTimeString(metrics.since)}.</p>`,
|
|
83
|
+
'<h2>Web page views</h2>',
|
|
84
|
+
getCategoryTable(metrics.pageViews),
|
|
85
|
+
'<h2>MCP tool calls</h2>',
|
|
86
|
+
getCategoryTable(metrics.mcpToolCalls),
|
|
87
|
+
'<h2>API operation calls</h2>',
|
|
88
|
+
getCategoryTable(metrics.apiOperations),
|
|
89
|
+
'<h2>Manager page activity</h2>',
|
|
90
|
+
'<p>Manager-only pages (linked from <a href="/manage.html">management</a>), excluded from the web page views above. A high failed count for a page may indicate a suspected attack, such as repeated guessing of the authorization code.</p>',
|
|
91
|
+
getManagerActivityTable(metrics.managerActivity)
|
|
92
|
+
].join('\n')
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
// Otherwise, i.e. if the form has not been submitted yet:
|
|
96
|
+
else {
|
|
97
|
+
query = {
|
|
98
|
+
body: [
|
|
99
|
+
'<form action="/metrics.html" method="post">',
|
|
100
|
+
' <p><label>',
|
|
101
|
+
' Authorization code: <input size="3" minlength="3" maxlength="3" name="authCode" required>',
|
|
102
|
+
' </label></p>',
|
|
103
|
+
' <p><label><input type="checkbox" name="clearCounts"> Clear counts</label></p>',
|
|
104
|
+
' <p><button type="submit">Submit</button></p>',
|
|
105
|
+
'</form>'
|
|
106
|
+
].join('\n')
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
// Get the populated template.
|
|
110
|
+
const answerPage = await populateTemplate(import.meta.dirname, query);
|
|
111
|
+
// Return the populated page. Submitting a valid authCode also sets a cookie that
|
|
112
|
+
// excludes this browser's future page views and API calls from usage metrics, on the
|
|
113
|
+
// premise that a maintainer who has just proven their identity here is very likely the
|
|
114
|
+
// same person about to manually browse and test the rest of Kilotest.
|
|
115
|
+
return {
|
|
116
|
+
status: 'ok',
|
|
117
|
+
answerPage,
|
|
118
|
+
...(method === 'POST' && {
|
|
119
|
+
setCookie: {
|
|
120
|
+
name: metricsExclusionCookieName,
|
|
121
|
+
value: getExclusionCookieValue(),
|
|
122
|
+
maxAgeSeconds: 30 * 24 * 60 * 60
|
|
123
|
+
}
|
|
124
|
+
})
|
|
125
|
+
};
|
|
126
|
+
};
|