@promptscore/core 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.cjs +1038 -0
- package/dist/browser.cjs.map +1 -0
- package/dist/browser.d.cts +14 -0
- package/dist/browser.d.ts +14 -0
- package/dist/browser.js +1024 -0
- package/dist/browser.js.map +1 -0
- package/dist/index-5jqZSlgq.d.cts +142 -0
- package/dist/index-5jqZSlgq.d.ts +142 -0
- package/dist/index.cjs +1136 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +35 -0
- package/dist/index.d.ts +35 -0
- package/dist/index.js +1120 -0
- package/dist/index.js.map +1 -0
- package/package.json +64 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,1120 @@
|
|
|
1
|
+
import { readFile } from 'fs/promises';
|
|
2
|
+
import { existsSync } from 'fs';
|
|
3
|
+
import { dirname, resolve, join } from 'path';
|
|
4
|
+
import { fileURLToPath } from 'url';
|
|
5
|
+
import { parse } from 'yaml';
|
|
6
|
+
|
|
7
|
+
// src/parser/index.ts
|
|
8
|
+
var ROLE_PATTERNS = [
|
|
9
|
+
/^you are\b[^.\n]*[.\n]/i,
|
|
10
|
+
/^you're\b[^.\n]*[.\n]/i,
|
|
11
|
+
/^act as\b[^.\n]*[.\n]/i,
|
|
12
|
+
/^your role is\b[^.\n]*[.\n]/i,
|
|
13
|
+
/^assume the role of\b[^.\n]*[.\n]/i,
|
|
14
|
+
/^imagine you are\b[^.\n]*[.\n]/i,
|
|
15
|
+
/^pretend to be\b[^.\n]*[.\n]/i
|
|
16
|
+
];
|
|
17
|
+
var TASK_KEYWORDS = [
|
|
18
|
+
/\byour task is\b/i,
|
|
19
|
+
/\byour job is\b/i,
|
|
20
|
+
/\byour goal is\b/i,
|
|
21
|
+
/\bplease\s+(?:write|explain|generate|analyze|summarize|translate|classify|extract|create|produce|list|describe)\b/i,
|
|
22
|
+
/\b(?:write|explain|generate|analyze|summarize|translate|classify|extract|create|produce|list|describe) (?:a|an|the)\b/i
|
|
23
|
+
];
|
|
24
|
+
var CONSTRAINT_KEYWORDS = [
|
|
25
|
+
/\b(?:must|should|should not|shouldn't|must not|mustn't|never|always|do not|don't|avoid|ensure|make sure|only)\b/i,
|
|
26
|
+
/\b(?:maximum|minimum|at most|at least|no more than|no less than|exactly)\b/i
|
|
27
|
+
];
|
|
28
|
+
var OUTPUT_FORMAT_KEYWORDS = [
|
|
29
|
+
/\b(?:output|respond|return|reply|answer)\s+(?:in|as|with|using)\s+(?:json|xml|yaml|markdown|html|csv|a list|a table|bullet points?|plain text|paragraphs?)\b/i,
|
|
30
|
+
/\bformat\s*:\s*(?:json|xml|yaml|markdown|html|csv|list|table)\b/i,
|
|
31
|
+
/\bin (?:json|xml|yaml|markdown|html|csv) format\b/i,
|
|
32
|
+
/\b(?:output|response|answer) format\b/i
|
|
33
|
+
];
|
|
34
|
+
var EXAMPLE_KEYWORDS = [
|
|
35
|
+
/\bfor example\b/i,
|
|
36
|
+
/\bexamples?:/i,
|
|
37
|
+
/\be\.g\./i,
|
|
38
|
+
/\bhere (?:is|are) (?:an? )?examples?\b/i,
|
|
39
|
+
/<example>/i,
|
|
40
|
+
/\binput\s*:[\s\S]*?output\s*:/i
|
|
41
|
+
];
|
|
42
|
+
var TONE_KEYWORDS = [
|
|
43
|
+
/\btone\s*:/i,
|
|
44
|
+
/\b(?:in a|use a) (?:formal|casual|friendly|professional|concise|humorous|playful|serious|polite) (?:tone|voice|style|manner)\b/i,
|
|
45
|
+
/\bbe (?:formal|casual|friendly|professional|concise|humorous|playful|serious|polite)\b/i,
|
|
46
|
+
/\bwrite in a .{1,20} (?:tone|voice|style)\b/i
|
|
47
|
+
];
|
|
48
|
+
var FALLBACK_KEYWORDS = [
|
|
49
|
+
/\bif (?:you (?:are )?(?:unsure|don't know|cannot|can't)|unsure|unclear|not sure|unable)\b/i,
|
|
50
|
+
/\bif you cannot\b/i,
|
|
51
|
+
/\bwhen in doubt\b/i,
|
|
52
|
+
/\bas a fallback\b/i,
|
|
53
|
+
/\bedge cases?\b/i
|
|
54
|
+
];
|
|
55
|
+
var CONTEXT_KEYWORDS = [
|
|
56
|
+
/^context\s*:/im,
|
|
57
|
+
/^background\s*:/im,
|
|
58
|
+
/^scenario\s*:/im,
|
|
59
|
+
/^given\s*:/im,
|
|
60
|
+
/<context>/i
|
|
61
|
+
];
|
|
62
|
+
var XML_TAG_REGEX = /<([a-z][a-z0-9_-]*)\b[^>]*>[\s\S]*?<\/\1>/i;
|
|
63
|
+
var MARKDOWN_HEADER_REGEX = /^#{1,6}\s+\S/m;
|
|
64
|
+
var NUMBERED_SECTION_REGEX = /^\s*(?:\d+[.)]\s+|-\s+|\*\s+)/m;
|
|
65
|
+
function countWords(text) {
|
|
66
|
+
const trimmed = text.trim();
|
|
67
|
+
if (!trimmed) return 0;
|
|
68
|
+
return trimmed.split(/\s+/).length;
|
|
69
|
+
}
|
|
70
|
+
function detectLanguage(text) {
|
|
71
|
+
const sample = text.toLowerCase().slice(0, 2e3);
|
|
72
|
+
const englishMarkers = /\b(?:the|and|you|your|that|this|with|for|are|not)\b/g;
|
|
73
|
+
const matches = sample.match(englishMarkers);
|
|
74
|
+
if (matches && matches.length >= 3) return "en";
|
|
75
|
+
return "unknown";
|
|
76
|
+
}
|
|
77
|
+
function extractFirstMatch(text, patterns) {
|
|
78
|
+
for (const pattern of patterns) {
|
|
79
|
+
const match = text.match(pattern);
|
|
80
|
+
if (match) {
|
|
81
|
+
return match[0].trim();
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
return void 0;
|
|
85
|
+
}
|
|
86
|
+
function extractRole(text) {
|
|
87
|
+
const firstLines = text.split(/\n/).slice(0, 3).join("\n");
|
|
88
|
+
for (const pattern of ROLE_PATTERNS) {
|
|
89
|
+
const match = firstLines.match(pattern);
|
|
90
|
+
if (match) return match[0].trim();
|
|
91
|
+
}
|
|
92
|
+
const xmlMatch = text.match(/<(?:role|persona)>([\s\S]*?)<\/(?:role|persona)>/i);
|
|
93
|
+
if (xmlMatch && xmlMatch[1]) return xmlMatch[1].trim();
|
|
94
|
+
return void 0;
|
|
95
|
+
}
|
|
96
|
+
function extractTask(text, role) {
|
|
97
|
+
const xmlMatch = text.match(
|
|
98
|
+
/<(?:task|instructions?|goal)>([\s\S]*?)<\/(?:task|instructions?|goal)>/i
|
|
99
|
+
);
|
|
100
|
+
if (xmlMatch && xmlMatch[1]) return xmlMatch[1].trim();
|
|
101
|
+
const withoutRole = role ? text.replace(role, "") : text;
|
|
102
|
+
const found = extractFirstMatch(withoutRole, TASK_KEYWORDS);
|
|
103
|
+
if (found) {
|
|
104
|
+
const idx = withoutRole.toLowerCase().indexOf(found.toLowerCase());
|
|
105
|
+
if (idx >= 0) {
|
|
106
|
+
const rest = withoutRole.slice(idx);
|
|
107
|
+
const end = rest.search(/[.!?\n]/);
|
|
108
|
+
return end > 0 ? rest.slice(0, end + 1).trim() : rest.slice(0, 200).trim();
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return void 0;
|
|
112
|
+
}
|
|
113
|
+
function extractContext(text) {
|
|
114
|
+
const xmlMatch = text.match(/<context>([\s\S]*?)<\/context>/i);
|
|
115
|
+
if (xmlMatch && xmlMatch[1]) return xmlMatch[1].trim();
|
|
116
|
+
for (const pattern of CONTEXT_KEYWORDS) {
|
|
117
|
+
const match = text.match(pattern);
|
|
118
|
+
if (match && match.index !== void 0) {
|
|
119
|
+
const rest = text.slice(match.index);
|
|
120
|
+
const end = rest.search(/\n\n/);
|
|
121
|
+
return end > 0 ? rest.slice(0, end).trim() : rest.slice(0, 300).trim();
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
return void 0;
|
|
125
|
+
}
|
|
126
|
+
function extractConstraints(text) {
|
|
127
|
+
const constraints = [];
|
|
128
|
+
const xmlMatch = text.match(/<constraints?>([\s\S]*?)<\/constraints?>/i);
|
|
129
|
+
if (xmlMatch && xmlMatch[1]) {
|
|
130
|
+
xmlMatch[1].split(/\n/).map((line) => line.replace(/^[\s\-*\d.)]+/, "").trim()).filter(Boolean).forEach((line) => constraints.push(line));
|
|
131
|
+
}
|
|
132
|
+
const lines = text.split(/\n/);
|
|
133
|
+
for (const line of lines) {
|
|
134
|
+
const trimmed = line.trim();
|
|
135
|
+
if (!trimmed) continue;
|
|
136
|
+
for (const pattern of CONSTRAINT_KEYWORDS) {
|
|
137
|
+
if (pattern.test(trimmed) && trimmed.length < 300) {
|
|
138
|
+
constraints.push(trimmed);
|
|
139
|
+
break;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
return Array.from(new Set(constraints));
|
|
144
|
+
}
|
|
145
|
+
function extractExamples(text) {
|
|
146
|
+
const examples = [];
|
|
147
|
+
const xmlRegex = /<example>([\s\S]*?)<\/example>/gi;
|
|
148
|
+
let xmlMatch;
|
|
149
|
+
while ((xmlMatch = xmlRegex.exec(text)) !== null) {
|
|
150
|
+
const body = xmlMatch[1]?.trim() ?? "";
|
|
151
|
+
const inputMatch = body.match(/input\s*:\s*([\s\S]*?)(?=output\s*:|$)/i);
|
|
152
|
+
const outputMatch = body.match(/output\s*:\s*([\s\S]*)/i);
|
|
153
|
+
examples.push({
|
|
154
|
+
raw: body,
|
|
155
|
+
input: inputMatch?.[1]?.trim(),
|
|
156
|
+
output: outputMatch?.[1]?.trim()
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
if (examples.length === 0) {
|
|
160
|
+
const ioRegex = /input\s*:\s*([\s\S]*?)\n\s*output\s*:\s*([\s\S]*?)(?=\n\s*input\s*:|\n\n|$)/gi;
|
|
161
|
+
let ioMatch;
|
|
162
|
+
while ((ioMatch = ioRegex.exec(text)) !== null) {
|
|
163
|
+
examples.push({
|
|
164
|
+
raw: ioMatch[0].trim(),
|
|
165
|
+
input: ioMatch[1]?.trim(),
|
|
166
|
+
output: ioMatch[2]?.trim()
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
if (examples.length === 0) {
|
|
171
|
+
for (const pattern of EXAMPLE_KEYWORDS) {
|
|
172
|
+
const match = text.match(pattern);
|
|
173
|
+
if (match && match.index !== void 0) {
|
|
174
|
+
const snippet = text.slice(match.index, match.index + 300);
|
|
175
|
+
examples.push({ raw: snippet.trim() });
|
|
176
|
+
break;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
return examples;
|
|
181
|
+
}
|
|
182
|
+
function extractOutputFormat(text) {
|
|
183
|
+
const xmlMatch = text.match(
|
|
184
|
+
/<(?:output|format|output_format)>([\s\S]*?)<\/(?:output|format|output_format)>/i
|
|
185
|
+
);
|
|
186
|
+
if (xmlMatch && xmlMatch[1]) return xmlMatch[1].trim();
|
|
187
|
+
return extractFirstMatch(text, OUTPUT_FORMAT_KEYWORDS);
|
|
188
|
+
}
|
|
189
|
+
function extractTone(text) {
|
|
190
|
+
return extractFirstMatch(text, TONE_KEYWORDS);
|
|
191
|
+
}
|
|
192
|
+
function extractFallback(text) {
|
|
193
|
+
return extractFirstMatch(text, FALLBACK_KEYWORDS);
|
|
194
|
+
}
|
|
195
|
+
function parsePrompt(raw) {
|
|
196
|
+
const text = raw;
|
|
197
|
+
const role = extractRole(text);
|
|
198
|
+
const task = extractTask(text, role);
|
|
199
|
+
const context = extractContext(text);
|
|
200
|
+
const constraints = extractConstraints(text);
|
|
201
|
+
const examples = extractExamples(text);
|
|
202
|
+
const outputFormat = extractOutputFormat(text);
|
|
203
|
+
const tone = extractTone(text);
|
|
204
|
+
const fallback = extractFallback(text);
|
|
205
|
+
const components = {
|
|
206
|
+
role,
|
|
207
|
+
task,
|
|
208
|
+
context,
|
|
209
|
+
constraints: constraints.length > 0 ? constraints : void 0,
|
|
210
|
+
examples: examples.length > 0 ? examples : void 0,
|
|
211
|
+
outputFormat,
|
|
212
|
+
tone,
|
|
213
|
+
fallback
|
|
214
|
+
};
|
|
215
|
+
const hasXmlTags = XML_TAG_REGEX.test(text);
|
|
216
|
+
const hasMarkdownHeaders = MARKDOWN_HEADER_REGEX.test(text);
|
|
217
|
+
const hasNumberedSections = NUMBERED_SECTION_REGEX.test(text);
|
|
218
|
+
const metadata = {
|
|
219
|
+
wordCount: countWords(text),
|
|
220
|
+
charCount: text.length,
|
|
221
|
+
lineCount: text.split(/\n/).length,
|
|
222
|
+
language: detectLanguage(text),
|
|
223
|
+
hasStructuredFormat: hasXmlTags || hasMarkdownHeaders || hasNumberedSections,
|
|
224
|
+
hasXmlTags,
|
|
225
|
+
hasMarkdownHeaders,
|
|
226
|
+
hasNumberedSections
|
|
227
|
+
};
|
|
228
|
+
return {
|
|
229
|
+
raw: text,
|
|
230
|
+
components,
|
|
231
|
+
metadata
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
// src/rules/deterministic/min-length.ts
|
|
236
|
+
var MIN_WORDS = 20;
|
|
237
|
+
var minLengthRule = {
|
|
238
|
+
id: "min-length",
|
|
239
|
+
name: "Minimum length",
|
|
240
|
+
description: "Prompts shorter than 20 words rarely give a model enough to work with.",
|
|
241
|
+
category: "specificity",
|
|
242
|
+
defaultSeverity: "warning",
|
|
243
|
+
type: "deterministic",
|
|
244
|
+
check: ({ ast }) => {
|
|
245
|
+
const wc = ast.metadata.wordCount;
|
|
246
|
+
const passed = wc >= MIN_WORDS;
|
|
247
|
+
const score = passed ? 100 : Math.round(wc / MIN_WORDS * 100);
|
|
248
|
+
return {
|
|
249
|
+
ruleId: "min-length",
|
|
250
|
+
passed,
|
|
251
|
+
score,
|
|
252
|
+
message: passed ? `Prompt length (${wc} words) is sufficient.` : `Prompt is very short (${wc} words). Models need enough context to respond well.`,
|
|
253
|
+
suggestion: passed ? void 0 : "Add more detail about what you want, why, and how the output should look.",
|
|
254
|
+
severity: "warning",
|
|
255
|
+
category: "specificity",
|
|
256
|
+
weight: 1
|
|
257
|
+
};
|
|
258
|
+
}
|
|
259
|
+
};
|
|
260
|
+
|
|
261
|
+
// src/rules/deterministic/max-length.ts
|
|
262
|
+
var SOFT_MAX = 1500;
|
|
263
|
+
var HARD_MAX = 3e3;
|
|
264
|
+
var maxLengthRule = {
|
|
265
|
+
id: "max-length",
|
|
266
|
+
name: "Maximum length",
|
|
267
|
+
description: "Excessively long prompts often contain redundancy and dilute the model\u2019s focus.",
|
|
268
|
+
category: "structure",
|
|
269
|
+
defaultSeverity: "info",
|
|
270
|
+
type: "deterministic",
|
|
271
|
+
check: ({ ast }) => {
|
|
272
|
+
const wc = ast.metadata.wordCount;
|
|
273
|
+
let score = 100;
|
|
274
|
+
if (wc > SOFT_MAX) {
|
|
275
|
+
const over = Math.min(wc - SOFT_MAX, HARD_MAX - SOFT_MAX);
|
|
276
|
+
score = Math.max(0, 100 - Math.round(over / (HARD_MAX - SOFT_MAX) * 100));
|
|
277
|
+
}
|
|
278
|
+
const passed = wc <= SOFT_MAX;
|
|
279
|
+
return {
|
|
280
|
+
ruleId: "max-length",
|
|
281
|
+
passed,
|
|
282
|
+
score,
|
|
283
|
+
message: passed ? `Prompt length (${wc} words) is within a comfortable range.` : `Prompt is long (${wc} words). Consider trimming redundancy or splitting into smaller prompts.`,
|
|
284
|
+
suggestion: passed ? void 0 : "Look for repeated instructions, bundled unrelated tasks, or sections that can be summarized.",
|
|
285
|
+
severity: "info",
|
|
286
|
+
category: "structure",
|
|
287
|
+
weight: 1
|
|
288
|
+
};
|
|
289
|
+
}
|
|
290
|
+
};
|
|
291
|
+
|
|
292
|
+
// src/rules/deterministic/no-output-format.ts
|
|
293
|
+
var noOutputFormatRule = {
|
|
294
|
+
id: "no-output-format",
|
|
295
|
+
name: "Output format specified",
|
|
296
|
+
description: "The prompt should tell the model exactly how to format its answer.",
|
|
297
|
+
category: "specificity",
|
|
298
|
+
defaultSeverity: "warning",
|
|
299
|
+
type: "deterministic",
|
|
300
|
+
check: ({ ast }) => {
|
|
301
|
+
const passed = Boolean(ast.components.outputFormat);
|
|
302
|
+
return {
|
|
303
|
+
ruleId: "no-output-format",
|
|
304
|
+
passed,
|
|
305
|
+
score: passed ? 100 : 0,
|
|
306
|
+
message: passed ? "An output format is specified." : "No output format is specified. The model will guess what you want.",
|
|
307
|
+
suggestion: passed ? void 0 : "State the exact format: JSON schema, bullet list, markdown table, single sentence, etc.",
|
|
308
|
+
severity: "warning",
|
|
309
|
+
category: "specificity",
|
|
310
|
+
weight: 1
|
|
311
|
+
};
|
|
312
|
+
}
|
|
313
|
+
};
|
|
314
|
+
|
|
315
|
+
// src/rules/deterministic/no-examples.ts
|
|
316
|
+
var noExamplesRule = {
|
|
317
|
+
id: "no-examples",
|
|
318
|
+
name: "Examples provided",
|
|
319
|
+
description: "Few-shot examples dramatically improve output quality and consistency.",
|
|
320
|
+
category: "best-practice",
|
|
321
|
+
defaultSeverity: "warning",
|
|
322
|
+
type: "deterministic",
|
|
323
|
+
check: ({ ast }) => {
|
|
324
|
+
const count = ast.components.examples?.length ?? 0;
|
|
325
|
+
const passed = count > 0;
|
|
326
|
+
return {
|
|
327
|
+
ruleId: "no-examples",
|
|
328
|
+
passed,
|
|
329
|
+
score: passed ? 100 : 0,
|
|
330
|
+
message: passed ? `Prompt includes ${count} example${count === 1 ? "" : "s"}.` : "No examples provided. Few-shot examples usually improve output quality.",
|
|
331
|
+
suggestion: passed ? void 0 : "Add 1\u20133 concrete examples showing the input and the expected output.",
|
|
332
|
+
severity: "warning",
|
|
333
|
+
category: "best-practice",
|
|
334
|
+
weight: 1
|
|
335
|
+
};
|
|
336
|
+
}
|
|
337
|
+
};
|
|
338
|
+
|
|
339
|
+
// src/rules/deterministic/no-role.ts
|
|
340
|
+
var noRoleRule = {
|
|
341
|
+
id: "no-role",
|
|
342
|
+
name: "Role or persona assigned",
|
|
343
|
+
description: "Assigning a role focuses the model and sets expectations for expertise and tone.",
|
|
344
|
+
category: "best-practice",
|
|
345
|
+
defaultSeverity: "info",
|
|
346
|
+
type: "deterministic",
|
|
347
|
+
check: ({ ast }) => {
|
|
348
|
+
const passed = Boolean(ast.components.role);
|
|
349
|
+
return {
|
|
350
|
+
ruleId: "no-role",
|
|
351
|
+
passed,
|
|
352
|
+
score: passed ? 100 : 0,
|
|
353
|
+
message: passed ? "A role or persona is assigned." : "No role assigned. Consider giving the model a persona aligned with the task.",
|
|
354
|
+
suggestion: passed ? void 0 : 'Start with something like "You are a senior <X> who specializes in <Y>."',
|
|
355
|
+
severity: "info",
|
|
356
|
+
category: "best-practice",
|
|
357
|
+
weight: 1
|
|
358
|
+
};
|
|
359
|
+
}
|
|
360
|
+
};
|
|
361
|
+
|
|
362
|
+
// src/rules/deterministic/no-context.ts
|
|
363
|
+
var noContextRule = {
|
|
364
|
+
id: "no-context",
|
|
365
|
+
name: "Background context provided",
|
|
366
|
+
description: "Context helps the model understand the situation, audience, and constraints.",
|
|
367
|
+
category: "specificity",
|
|
368
|
+
defaultSeverity: "info",
|
|
369
|
+
type: "deterministic",
|
|
370
|
+
check: ({ ast }) => {
|
|
371
|
+
const passed = Boolean(ast.components.context) || ast.metadata.wordCount >= 80;
|
|
372
|
+
return {
|
|
373
|
+
ruleId: "no-context",
|
|
374
|
+
passed,
|
|
375
|
+
score: passed ? 100 : 40,
|
|
376
|
+
message: passed ? "Prompt provides context or is detailed enough to imply it." : "No background context detected. The model may miss important situational cues.",
|
|
377
|
+
suggestion: passed ? void 0 : "Explain the situation: who the user is, why they\u2019re asking, and what the stakes are.",
|
|
378
|
+
severity: "info",
|
|
379
|
+
category: "specificity",
|
|
380
|
+
weight: 1
|
|
381
|
+
};
|
|
382
|
+
}
|
|
383
|
+
};
|
|
384
|
+
|
|
385
|
+
// src/rules/deterministic/ambiguous-negation.ts
|
|
386
|
+
var NEGATION_PATTERN = /\b(?:don't|do not|never|avoid|shouldn't|should not|must not|mustn't)\b/gi;
|
|
387
|
+
var ambiguousNegationRule = {
|
|
388
|
+
id: "ambiguous-negation",
|
|
389
|
+
name: "Ambiguous negative instructions",
|
|
390
|
+
description: `Negative instructions ("don't do X") are less effective than positive ones ("do Y").`,
|
|
391
|
+
category: "clarity",
|
|
392
|
+
defaultSeverity: "info",
|
|
393
|
+
type: "deterministic",
|
|
394
|
+
check: ({ ast }) => {
|
|
395
|
+
const matches = ast.raw.match(NEGATION_PATTERN) ?? [];
|
|
396
|
+
const wc = ast.metadata.wordCount || 1;
|
|
397
|
+
const ratio = matches.length / wc;
|
|
398
|
+
const passed = matches.length < 3 || ratio < 0.05;
|
|
399
|
+
const score = passed ? 100 : Math.max(20, 100 - matches.length * 10);
|
|
400
|
+
return {
|
|
401
|
+
ruleId: "ambiguous-negation",
|
|
402
|
+
passed,
|
|
403
|
+
score,
|
|
404
|
+
message: passed ? `Negative instructions are used sparingly (${matches.length} found).` : `Heavy use of negative instructions (${matches.length} found). Models follow positive instructions more reliably.`,
|
|
405
|
+
suggestion: passed ? void 0 : `Rewrite "don't do X" as "do Y instead". Tell the model what the desired behavior is.`,
|
|
406
|
+
severity: "info",
|
|
407
|
+
category: "clarity",
|
|
408
|
+
weight: 1
|
|
409
|
+
};
|
|
410
|
+
}
|
|
411
|
+
};
|
|
412
|
+
|
|
413
|
+
// src/rules/deterministic/no-constraints.ts
|
|
414
|
+
var noConstraintsRule = {
|
|
415
|
+
id: "no-constraints",
|
|
416
|
+
name: "Constraints defined",
|
|
417
|
+
description: "Explicit constraints keep the model on track and prevent scope drift.",
|
|
418
|
+
category: "specificity",
|
|
419
|
+
defaultSeverity: "info",
|
|
420
|
+
type: "deterministic",
|
|
421
|
+
check: ({ ast }) => {
|
|
422
|
+
const count = ast.components.constraints?.length ?? 0;
|
|
423
|
+
const passed = count > 0;
|
|
424
|
+
return {
|
|
425
|
+
ruleId: "no-constraints",
|
|
426
|
+
passed,
|
|
427
|
+
score: passed ? 100 : 0,
|
|
428
|
+
message: passed ? `Prompt defines ${count} constraint${count === 1 ? "" : "s"}.` : "No constraints detected. The model has no guardrails on length, scope, or style.",
|
|
429
|
+
suggestion: passed ? void 0 : "Add constraints like length limits, scope boundaries, or things the answer must include.",
|
|
430
|
+
severity: "info",
|
|
431
|
+
category: "specificity",
|
|
432
|
+
weight: 1
|
|
433
|
+
};
|
|
434
|
+
}
|
|
435
|
+
};
|
|
436
|
+
|
|
437
|
+
// src/rules/deterministic/all-caps-abuse.ts
|
|
438
|
+
var CAPS_WORD_PATTERN = /\b[A-Z]{4,}\b/g;
|
|
439
|
+
var allCapsAbuseRule = {
|
|
440
|
+
id: "all-caps-abuse",
|
|
441
|
+
name: "All-caps abuse",
|
|
442
|
+
description: "Excessive ALL CAPS is noisy and rarely the clearest way to emphasize something.",
|
|
443
|
+
category: "clarity",
|
|
444
|
+
defaultSeverity: "info",
|
|
445
|
+
type: "deterministic",
|
|
446
|
+
check: ({ ast }) => {
|
|
447
|
+
const caps = ast.raw.match(CAPS_WORD_PATTERN) ?? [];
|
|
448
|
+
const wc = ast.metadata.wordCount || 1;
|
|
449
|
+
const ratio = caps.length / wc;
|
|
450
|
+
const passed = ratio < 0.05 && caps.length < 8;
|
|
451
|
+
const score = passed ? 100 : Math.max(30, 100 - Math.round(ratio * 1e3));
|
|
452
|
+
return {
|
|
453
|
+
ruleId: "all-caps-abuse",
|
|
454
|
+
passed,
|
|
455
|
+
score,
|
|
456
|
+
message: passed ? "All-caps usage is within reasonable limits." : `Excessive ALL CAPS (${caps.length} words). This rarely improves comprehension.`,
|
|
457
|
+
suggestion: passed ? void 0 : "Use bold (**word**), quotes, or XML tags for emphasis instead of ALL CAPS.",
|
|
458
|
+
severity: "info",
|
|
459
|
+
category: "clarity",
|
|
460
|
+
weight: 1
|
|
461
|
+
};
|
|
462
|
+
}
|
|
463
|
+
};
|
|
464
|
+
|
|
465
|
+
// src/rules/deterministic/vague-instruction.ts
|
|
466
|
+
var VAGUE_WORDS = [
|
|
467
|
+
"good",
|
|
468
|
+
"nice",
|
|
469
|
+
"proper",
|
|
470
|
+
"appropriate",
|
|
471
|
+
"correct",
|
|
472
|
+
"better",
|
|
473
|
+
"high quality",
|
|
474
|
+
"high-quality",
|
|
475
|
+
"reasonable",
|
|
476
|
+
"decent",
|
|
477
|
+
"suitable"
|
|
478
|
+
];
|
|
479
|
+
function buildVaguePattern() {
|
|
480
|
+
const escaped = VAGUE_WORDS.map((w) => w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
|
|
481
|
+
return new RegExp(`\\b(?:${escaped.join("|")})\\b`, "gi");
|
|
482
|
+
}
|
|
483
|
+
var VAGUE_PATTERN = buildVaguePattern();
|
|
484
|
+
var vagueInstructionRule = {
|
|
485
|
+
id: "vague-instruction",
|
|
486
|
+
name: "Vague instructions",
|
|
487
|
+
description: 'Vague words like "good", "proper", or "appropriate" do not give the model a concrete target.',
|
|
488
|
+
category: "clarity",
|
|
489
|
+
defaultSeverity: "warning",
|
|
490
|
+
type: "deterministic",
|
|
491
|
+
check: ({ ast }) => {
|
|
492
|
+
const matches = ast.raw.match(VAGUE_PATTERN) ?? [];
|
|
493
|
+
const unique = Array.from(new Set(matches.map((m) => m.toLowerCase())));
|
|
494
|
+
const passed = matches.length === 0;
|
|
495
|
+
const score = passed ? 100 : Math.max(25, 100 - matches.length * 15);
|
|
496
|
+
return {
|
|
497
|
+
ruleId: "vague-instruction",
|
|
498
|
+
passed,
|
|
499
|
+
score,
|
|
500
|
+
message: passed ? "No vague qualifiers detected." : `Vague qualifiers used: ${unique.join(", ")}.`,
|
|
501
|
+
suggestion: passed ? void 0 : 'Replace vague words with measurable criteria. "Good" \u2192 "concise (\u2264 3 sentences) and citing sources".',
|
|
502
|
+
severity: "warning",
|
|
503
|
+
category: "clarity",
|
|
504
|
+
weight: 1
|
|
505
|
+
};
|
|
506
|
+
}
|
|
507
|
+
};
|
|
508
|
+
|
|
509
|
+
// src/rules/deterministic/missing-task.ts
|
|
510
|
+
var missingTaskRule = {
|
|
511
|
+
id: "missing-task",
|
|
512
|
+
name: "Clear task or instruction",
|
|
513
|
+
description: "Every prompt should have an unambiguous instruction telling the model what to do.",
|
|
514
|
+
category: "clarity",
|
|
515
|
+
defaultSeverity: "error",
|
|
516
|
+
type: "deterministic",
|
|
517
|
+
check: ({ ast }) => {
|
|
518
|
+
const passed = Boolean(ast.components.task);
|
|
519
|
+
return {
|
|
520
|
+
ruleId: "missing-task",
|
|
521
|
+
passed,
|
|
522
|
+
score: passed ? 100 : 0,
|
|
523
|
+
message: passed ? "A clear task is present." : "No explicit task detected. The model may not know what you want it to do.",
|
|
524
|
+
suggestion: passed ? void 0 : 'State the task explicitly: "Your task is to..." or "Please <verb> <object>".',
|
|
525
|
+
severity: "error",
|
|
526
|
+
category: "clarity",
|
|
527
|
+
weight: 1.5
|
|
528
|
+
};
|
|
529
|
+
}
|
|
530
|
+
};
|
|
531
|
+
|
|
532
|
+
// src/rules/deterministic/no-structured-format.ts
|
|
533
|
+
var LONG_PROMPT_THRESHOLD = 100;
|
|
534
|
+
var noStructuredFormatRule = {
|
|
535
|
+
id: "no-structured-format",
|
|
536
|
+
name: "Structured formatting",
|
|
537
|
+
description: "Long prompts are easier for a model to follow when broken into sections (XML tags, headers, lists).",
|
|
538
|
+
category: "structure",
|
|
539
|
+
defaultSeverity: "warning",
|
|
540
|
+
type: "deterministic",
|
|
541
|
+
check: ({ ast }) => {
|
|
542
|
+
const wc = ast.metadata.wordCount;
|
|
543
|
+
const structured = ast.metadata.hasStructuredFormat;
|
|
544
|
+
const passed = wc < LONG_PROMPT_THRESHOLD || structured;
|
|
545
|
+
const score = passed ? 100 : 40;
|
|
546
|
+
return {
|
|
547
|
+
ruleId: "no-structured-format",
|
|
548
|
+
passed,
|
|
549
|
+
score,
|
|
550
|
+
message: passed ? structured ? "Prompt uses structured formatting." : "Prompt is short enough to not need structural formatting." : `Long prompt (${wc} words) without structural markers like XML tags, headers, or numbered sections.`,
|
|
551
|
+
suggestion: passed ? void 0 : "Split the prompt into labeled sections: <instructions>, <context>, <examples>, <output_format>.",
|
|
552
|
+
severity: "warning",
|
|
553
|
+
category: "structure",
|
|
554
|
+
weight: 1
|
|
555
|
+
};
|
|
556
|
+
}
|
|
557
|
+
};
|
|
558
|
+
|
|
559
|
+
// src/rules/deterministic/index.ts
|
|
560
|
+
var deterministicRules = [
|
|
561
|
+
minLengthRule,
|
|
562
|
+
maxLengthRule,
|
|
563
|
+
noOutputFormatRule,
|
|
564
|
+
noExamplesRule,
|
|
565
|
+
noRoleRule,
|
|
566
|
+
noContextRule,
|
|
567
|
+
ambiguousNegationRule,
|
|
568
|
+
noConstraintsRule,
|
|
569
|
+
allCapsAbuseRule,
|
|
570
|
+
vagueInstructionRule,
|
|
571
|
+
missingTaskRule,
|
|
572
|
+
noStructuredFormatRule
|
|
573
|
+
];
|
|
574
|
+
|
|
575
|
+
// src/rules/registry.ts
|
|
576
|
+
var RuleRegistry = class {
|
|
577
|
+
rules = /* @__PURE__ */ new Map();
|
|
578
|
+
constructor(initialRules = []) {
|
|
579
|
+
for (const rule of initialRules) {
|
|
580
|
+
this.register(rule);
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
register(rule) {
|
|
584
|
+
if (this.rules.has(rule.id)) {
|
|
585
|
+
throw new Error(`Rule with id "${rule.id}" is already registered`);
|
|
586
|
+
}
|
|
587
|
+
this.rules.set(rule.id, rule);
|
|
588
|
+
}
|
|
589
|
+
get(id) {
|
|
590
|
+
return this.rules.get(id);
|
|
591
|
+
}
|
|
592
|
+
has(id) {
|
|
593
|
+
return this.rules.has(id);
|
|
594
|
+
}
|
|
595
|
+
all() {
|
|
596
|
+
return Array.from(this.rules.values());
|
|
597
|
+
}
|
|
598
|
+
byType(type) {
|
|
599
|
+
return this.all().filter((rule) => rule.type === type);
|
|
600
|
+
}
|
|
601
|
+
byIds(ids) {
|
|
602
|
+
const out = [];
|
|
603
|
+
for (const id of ids) {
|
|
604
|
+
const rule = this.rules.get(id);
|
|
605
|
+
if (rule) out.push(rule);
|
|
606
|
+
}
|
|
607
|
+
return out;
|
|
608
|
+
}
|
|
609
|
+
};
|
|
610
|
+
function createDefaultRegistry() {
|
|
611
|
+
return new RuleRegistry(deterministicRules);
|
|
612
|
+
}
|
|
613
|
+
var __filename$1 = fileURLToPath(import.meta.url);
|
|
614
|
+
var __dirname$1 = dirname(__filename$1);
|
|
615
|
+
function findProfilesDir(startDir) {
|
|
616
|
+
let current = startDir;
|
|
617
|
+
for (let i = 0; i < 6; i++) {
|
|
618
|
+
const candidate = join(current, "profiles");
|
|
619
|
+
if (existsSync(candidate)) return candidate;
|
|
620
|
+
const parent = dirname(current);
|
|
621
|
+
if (parent === current) break;
|
|
622
|
+
current = parent;
|
|
623
|
+
}
|
|
624
|
+
return void 0;
|
|
625
|
+
}
|
|
626
|
+
function normalize(raw, name) {
|
|
627
|
+
return {
|
|
628
|
+
name: raw.name ?? name,
|
|
629
|
+
displayName: raw.display_name ?? raw.name ?? name,
|
|
630
|
+
version: raw.version,
|
|
631
|
+
base: raw.base,
|
|
632
|
+
rules: raw.rules ?? {},
|
|
633
|
+
bestPractices: raw.best_practices ?? []
|
|
634
|
+
};
|
|
635
|
+
}
|
|
636
|
+
function mergeOverrides(base, override) {
|
|
637
|
+
return {
|
|
638
|
+
severity: override?.severity ?? base?.severity,
|
|
639
|
+
weight: override?.weight ?? base?.weight,
|
|
640
|
+
suggestion: override?.suggestion ?? base?.suggestion,
|
|
641
|
+
reference: override?.reference ?? base?.reference,
|
|
642
|
+
enabled: override?.enabled ?? base?.enabled
|
|
643
|
+
};
|
|
644
|
+
}
|
|
645
|
+
function mergeProfiles(base, child) {
|
|
646
|
+
const ruleIds = /* @__PURE__ */ new Set([...Object.keys(base.rules), ...Object.keys(child.rules)]);
|
|
647
|
+
const merged = {};
|
|
648
|
+
for (const id of ruleIds) {
|
|
649
|
+
merged[id] = mergeOverrides(base.rules[id], child.rules[id]);
|
|
650
|
+
}
|
|
651
|
+
return {
|
|
652
|
+
name: child.name,
|
|
653
|
+
displayName: child.displayName,
|
|
654
|
+
version: child.version ?? base.version,
|
|
655
|
+
base: child.base,
|
|
656
|
+
rules: merged,
|
|
657
|
+
bestPractices: [...base.bestPractices, ...child.bestPractices]
|
|
658
|
+
};
|
|
659
|
+
}
|
|
660
|
+
var ProfileLoader = class {
|
|
661
|
+
profilesDir;
|
|
662
|
+
cache = /* @__PURE__ */ new Map();
|
|
663
|
+
constructor(options = {}) {
|
|
664
|
+
const found = options.profilesDir ?? findProfilesDir(__dirname$1) ?? findProfilesDir(process.cwd());
|
|
665
|
+
if (!found) {
|
|
666
|
+
throw new Error("Could not locate a `profiles/` directory. Pass `profilesDir` explicitly.");
|
|
667
|
+
}
|
|
668
|
+
this.profilesDir = resolve(found);
|
|
669
|
+
}
|
|
670
|
+
get directory() {
|
|
671
|
+
return this.profilesDir;
|
|
672
|
+
}
|
|
673
|
+
async load(name) {
|
|
674
|
+
if (this.cache.has(name)) {
|
|
675
|
+
return this.cache.get(name);
|
|
676
|
+
}
|
|
677
|
+
const filePath = join(this.profilesDir, `${name}.yaml`);
|
|
678
|
+
if (!existsSync(filePath)) {
|
|
679
|
+
throw new Error(`Profile "${name}" not found at ${filePath}`);
|
|
680
|
+
}
|
|
681
|
+
const content = await readFile(filePath, "utf8");
|
|
682
|
+
const raw = parse(content) ?? {};
|
|
683
|
+
let profile = normalize(raw, name);
|
|
684
|
+
if (profile.base) {
|
|
685
|
+
const base = await this.load(profile.base);
|
|
686
|
+
profile = mergeProfiles(base, profile);
|
|
687
|
+
}
|
|
688
|
+
this.cache.set(name, profile);
|
|
689
|
+
return profile;
|
|
690
|
+
}
|
|
691
|
+
async list() {
|
|
692
|
+
const { readdir } = await import('fs/promises');
|
|
693
|
+
const files = await readdir(this.profilesDir);
|
|
694
|
+
return files.filter((f) => f.endsWith(".yaml") && !f.startsWith("_")).map((f) => f.replace(/\.yaml$/, ""));
|
|
695
|
+
}
|
|
696
|
+
};
|
|
697
|
+
|
|
698
|
+
// src/scorer/index.ts
|
|
699
|
+
function isRuleEnabled(rule, profile) {
|
|
700
|
+
const override = profile.rules[rule.id];
|
|
701
|
+
if (override?.enabled === false) return false;
|
|
702
|
+
return true;
|
|
703
|
+
}
|
|
704
|
+
function applyOverride(result, profile) {
|
|
705
|
+
const override = profile.rules[result.ruleId];
|
|
706
|
+
if (!override) return result;
|
|
707
|
+
return {
|
|
708
|
+
...result,
|
|
709
|
+
severity: override.severity ?? result.severity,
|
|
710
|
+
weight: override.weight ?? result.weight,
|
|
711
|
+
suggestion: override.suggestion ?? result.suggestion,
|
|
712
|
+
reference: override.reference ?? result.reference
|
|
713
|
+
};
|
|
714
|
+
}
|
|
715
|
+
async function runRules(options) {
|
|
716
|
+
const { rules, profile, ast, only, includeLlm } = options;
|
|
717
|
+
const results = [];
|
|
718
|
+
for (const rule of rules) {
|
|
719
|
+
if (only && !only.includes(rule.id)) continue;
|
|
720
|
+
if (!isRuleEnabled(rule, profile)) continue;
|
|
721
|
+
if (rule.type === "llm" && !includeLlm) continue;
|
|
722
|
+
const base = await rule.check({ ast, profile });
|
|
723
|
+
results.push(applyOverride(base, profile));
|
|
724
|
+
}
|
|
725
|
+
return results;
|
|
726
|
+
}
|
|
727
|
+
function groupByCategory(results) {
|
|
728
|
+
const grouped = /* @__PURE__ */ new Map();
|
|
729
|
+
for (const result of results) {
|
|
730
|
+
const existing = grouped.get(result.category) ?? [];
|
|
731
|
+
existing.push(result);
|
|
732
|
+
grouped.set(result.category, existing);
|
|
733
|
+
}
|
|
734
|
+
return grouped;
|
|
735
|
+
}
|
|
736
|
+
function weightedAverage(values, weights) {
|
|
737
|
+
const totalWeight = weights.reduce((a, b) => a + b, 0);
|
|
738
|
+
if (totalWeight === 0) return 0;
|
|
739
|
+
const sum = values.reduce((acc, v, i) => acc + v * (weights[i] ?? 1), 0);
|
|
740
|
+
return sum / totalWeight;
|
|
741
|
+
}
|
|
742
|
+
function buildSummary(overall, results) {
|
|
743
|
+
const failed = results.filter((r) => !r.passed);
|
|
744
|
+
if (failed.length === 0) return `Score ${Math.round(overall)}/100 \u2014 strong prompt.`;
|
|
745
|
+
const errors = failed.filter((r) => r.severity === "error").length;
|
|
746
|
+
const warnings = failed.filter((r) => r.severity === "warning").length;
|
|
747
|
+
const pieces = [];
|
|
748
|
+
if (errors) pieces.push(`${errors} error${errors === 1 ? "" : "s"}`);
|
|
749
|
+
if (warnings) pieces.push(`${warnings} warning${warnings === 1 ? "" : "s"}`);
|
|
750
|
+
const extras = failed.length - errors - warnings;
|
|
751
|
+
if (extras > 0) pieces.push(`${extras} info`);
|
|
752
|
+
return `Score ${Math.round(overall)}/100 \u2014 ${pieces.join(", ")}.`;
|
|
753
|
+
}
|
|
754
|
+
function buildSuggestions(results) {
|
|
755
|
+
const severityRank = {
|
|
756
|
+
error: 3,
|
|
757
|
+
warning: 2,
|
|
758
|
+
info: 1
|
|
759
|
+
};
|
|
760
|
+
return results.filter((r) => !r.passed && r.suggestion).map((r) => ({
|
|
761
|
+
ruleId: r.ruleId,
|
|
762
|
+
severity: r.severity,
|
|
763
|
+
message: r.message,
|
|
764
|
+
suggestion: r.suggestion,
|
|
765
|
+
reference: r.reference,
|
|
766
|
+
impact: (100 - r.score) * r.weight * severityRank[r.severity]
|
|
767
|
+
})).sort((a, b) => b.impact - a.impact);
|
|
768
|
+
}
|
|
769
|
+
function buildReport(results, profile) {
|
|
770
|
+
const grouped = groupByCategory(results);
|
|
771
|
+
const categories = [];
|
|
772
|
+
for (const [category, rs] of grouped) {
|
|
773
|
+
const score = weightedAverage(
|
|
774
|
+
rs.map((r) => r.score),
|
|
775
|
+
rs.map((r) => r.weight)
|
|
776
|
+
);
|
|
777
|
+
categories.push({ category, score, rules: rs });
|
|
778
|
+
}
|
|
779
|
+
const overall = weightedAverage(
|
|
780
|
+
categories.map((c2) => c2.score),
|
|
781
|
+
categories.map((c2) => c2.rules.reduce((a, r) => a + r.weight, 0))
|
|
782
|
+
);
|
|
783
|
+
return {
|
|
784
|
+
overall,
|
|
785
|
+
categories,
|
|
786
|
+
summary: buildSummary(overall, results),
|
|
787
|
+
suggestions: buildSuggestions(results),
|
|
788
|
+
results,
|
|
789
|
+
profileName: profile.name
|
|
790
|
+
};
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
// src/profiles/builtin.ts
|
|
794
|
+
function mergeOverrides2(base, override) {
|
|
795
|
+
return {
|
|
796
|
+
severity: override?.severity ?? base?.severity,
|
|
797
|
+
weight: override?.weight ?? base?.weight,
|
|
798
|
+
suggestion: override?.suggestion ?? base?.suggestion,
|
|
799
|
+
reference: override?.reference ?? base?.reference,
|
|
800
|
+
enabled: override?.enabled ?? base?.enabled
|
|
801
|
+
};
|
|
802
|
+
}
|
|
803
|
+
function mergeProfiles2(base, child) {
|
|
804
|
+
const ruleIds = /* @__PURE__ */ new Set([...Object.keys(base.rules), ...Object.keys(child.rules)]);
|
|
805
|
+
const rules = {};
|
|
806
|
+
for (const ruleId of ruleIds) {
|
|
807
|
+
rules[ruleId] = mergeOverrides2(base.rules[ruleId], child.rules[ruleId]);
|
|
808
|
+
}
|
|
809
|
+
return {
|
|
810
|
+
name: child.name,
|
|
811
|
+
displayName: child.displayName,
|
|
812
|
+
version: child.version ?? base.version,
|
|
813
|
+
base: child.base,
|
|
814
|
+
rules,
|
|
815
|
+
bestPractices: [...base.bestPractices, ...child.bestPractices]
|
|
816
|
+
};
|
|
817
|
+
}
|
|
818
|
+
var baseProfile = {
|
|
819
|
+
name: "_base",
|
|
820
|
+
displayName: "Model-agnostic baseline",
|
|
821
|
+
version: "2026-04",
|
|
822
|
+
base: void 0,
|
|
823
|
+
rules: {
|
|
824
|
+
"min-length": {
|
|
825
|
+
severity: "warning",
|
|
826
|
+
weight: 1
|
|
827
|
+
},
|
|
828
|
+
"max-length": {
|
|
829
|
+
severity: "info",
|
|
830
|
+
weight: 0.8
|
|
831
|
+
},
|
|
832
|
+
"no-output-format": {
|
|
833
|
+
severity: "warning",
|
|
834
|
+
weight: 1.2,
|
|
835
|
+
suggestion: "Tell the model the exact format you want (JSON schema, bullet list, single sentence, etc.)."
|
|
836
|
+
},
|
|
837
|
+
"no-examples": {
|
|
838
|
+
severity: "warning",
|
|
839
|
+
weight: 1,
|
|
840
|
+
suggestion: "Add 1\u20133 concrete examples showing the input and the expected output."
|
|
841
|
+
},
|
|
842
|
+
"no-role": {
|
|
843
|
+
severity: "info",
|
|
844
|
+
weight: 0.8
|
|
845
|
+
},
|
|
846
|
+
"no-context": {
|
|
847
|
+
severity: "info",
|
|
848
|
+
weight: 0.9
|
|
849
|
+
},
|
|
850
|
+
"ambiguous-negation": {
|
|
851
|
+
severity: "info",
|
|
852
|
+
weight: 0.8
|
|
853
|
+
},
|
|
854
|
+
"no-constraints": {
|
|
855
|
+
severity: "info",
|
|
856
|
+
weight: 0.9
|
|
857
|
+
},
|
|
858
|
+
"all-caps-abuse": {
|
|
859
|
+
severity: "info",
|
|
860
|
+
weight: 0.6
|
|
861
|
+
},
|
|
862
|
+
"vague-instruction": {
|
|
863
|
+
severity: "warning",
|
|
864
|
+
weight: 1.1
|
|
865
|
+
},
|
|
866
|
+
"missing-task": {
|
|
867
|
+
severity: "error",
|
|
868
|
+
weight: 2
|
|
869
|
+
},
|
|
870
|
+
"no-structured-format": {
|
|
871
|
+
severity: "warning",
|
|
872
|
+
weight: 1
|
|
873
|
+
}
|
|
874
|
+
},
|
|
875
|
+
bestPractices: [
|
|
876
|
+
"Be explicit about the task \u2014 one clear instruction beats several vague ones.",
|
|
877
|
+
"Include at least one concrete example when the output format matters.",
|
|
878
|
+
"Prefer positive instructions ('do Y') over negative ones ('don't do X').",
|
|
879
|
+
"Specify the exact output format the model should return.",
|
|
880
|
+
"Break long prompts into labeled sections."
|
|
881
|
+
]
|
|
882
|
+
};
|
|
883
|
+
var claudeOverrides = {
|
|
884
|
+
name: "claude",
|
|
885
|
+
displayName: "Anthropic Claude",
|
|
886
|
+
version: "2026-04",
|
|
887
|
+
base: "_base",
|
|
888
|
+
rules: {
|
|
889
|
+
"no-output-format": {
|
|
890
|
+
severity: "warning",
|
|
891
|
+
weight: 1.2,
|
|
892
|
+
suggestion: "Claude responds well to XML-tagged output format instructions like <output_format>.",
|
|
893
|
+
reference: "https://docs.anthropic.com/en/docs/build-with-claude/prompt-engineering"
|
|
894
|
+
},
|
|
895
|
+
"no-examples": {
|
|
896
|
+
severity: "warning",
|
|
897
|
+
weight: 1.5,
|
|
898
|
+
suggestion: "Claude benefits significantly from 2\u20133 few-shot examples inside <example> tags.",
|
|
899
|
+
reference: "https://docs.anthropic.com/en/docs/build-with-claude/prompt-engineering/multishot-prompting"
|
|
900
|
+
},
|
|
901
|
+
"no-structured-format": {
|
|
902
|
+
severity: "warning",
|
|
903
|
+
weight: 1.3,
|
|
904
|
+
suggestion: "Claude handles XML tags particularly well for separating sections.",
|
|
905
|
+
reference: "https://docs.anthropic.com/en/docs/build-with-claude/prompt-engineering/use-xml-tags"
|
|
906
|
+
},
|
|
907
|
+
"ambiguous-negation": {
|
|
908
|
+
severity: "warning",
|
|
909
|
+
weight: 1.1,
|
|
910
|
+
suggestion: "Claude follows positive framings more reliably than negations."
|
|
911
|
+
}
|
|
912
|
+
},
|
|
913
|
+
bestPractices: [
|
|
914
|
+
"Use XML tags to separate sections: <instructions>, <context>, <example>, <output_format>.",
|
|
915
|
+
"Put the most important instructions at both the beginning and the end of the prompt.",
|
|
916
|
+
"Use positive instructions ('do X') rather than negative ('don't do Y').",
|
|
917
|
+
"Provide 2\u20133 examples of the expected output format.",
|
|
918
|
+
"For long prompts, wrap each section in an XML tag so Claude can refer back to it."
|
|
919
|
+
]
|
|
920
|
+
};
|
|
921
|
+
var gptOverrides = {
|
|
922
|
+
name: "gpt",
|
|
923
|
+
displayName: "OpenAI GPT",
|
|
924
|
+
version: "2026-04",
|
|
925
|
+
base: "_base",
|
|
926
|
+
rules: {
|
|
927
|
+
"no-output-format": {
|
|
928
|
+
severity: "warning",
|
|
929
|
+
weight: 1.3,
|
|
930
|
+
suggestion: "GPT responds well to explicit format instructions \u2014 mention JSON schema or a numbered list if relevant.",
|
|
931
|
+
reference: "https://platform.openai.com/docs/guides/prompt-engineering"
|
|
932
|
+
},
|
|
933
|
+
"no-examples": {
|
|
934
|
+
severity: "warning",
|
|
935
|
+
weight: 1.3,
|
|
936
|
+
suggestion: "Use a system + user pattern with few-shot examples to anchor GPT\u2019s behavior."
|
|
937
|
+
},
|
|
938
|
+
"no-structured-format": {
|
|
939
|
+
severity: "warning",
|
|
940
|
+
weight: 1.1,
|
|
941
|
+
suggestion: "Markdown headers and numbered sections work well for GPT."
|
|
942
|
+
},
|
|
943
|
+
"no-role": {
|
|
944
|
+
severity: "info",
|
|
945
|
+
weight: 1,
|
|
946
|
+
suggestion: "GPT responds well when given an explicit persona via a system message."
|
|
947
|
+
}
|
|
948
|
+
},
|
|
949
|
+
bestPractices: [
|
|
950
|
+
"Use a system message to set the persona and global instructions.",
|
|
951
|
+
"Structure long prompts with markdown headers (## Context, ## Task, ## Output).",
|
|
952
|
+
"For structured output, prefer JSON schema or response_format: json_object.",
|
|
953
|
+
"Include a handful of few-shot examples if format matters.",
|
|
954
|
+
"Be explicit about what the model should not do, but always pair with what it should do."
|
|
955
|
+
]
|
|
956
|
+
};
|
|
957
|
+
var builtinProfiles = {
|
|
958
|
+
_base: baseProfile,
|
|
959
|
+
claude: mergeProfiles2(baseProfile, claudeOverrides),
|
|
960
|
+
gpt: mergeProfiles2(baseProfile, gptOverrides)
|
|
961
|
+
};
|
|
962
|
+
function getBuiltinProfile(name) {
|
|
963
|
+
return structuredClone(builtinProfiles[name]);
|
|
964
|
+
}
|
|
965
|
+
|
|
966
|
+
// src/reporter/index.ts
|
|
967
|
+
var COLORS = {
|
|
968
|
+
reset: "\x1B[0m",
|
|
969
|
+
bold: "\x1B[1m",
|
|
970
|
+
dim: "\x1B[2m",
|
|
971
|
+
red: "\x1B[31m",
|
|
972
|
+
green: "\x1B[32m",
|
|
973
|
+
yellow: "\x1B[33m",
|
|
974
|
+
blue: "\x1B[34m",
|
|
975
|
+
magenta: "\x1B[35m",
|
|
976
|
+
cyan: "\x1B[36m",
|
|
977
|
+
gray: "\x1B[90m"
|
|
978
|
+
};
|
|
979
|
+
function c(color, text, enabled) {
|
|
980
|
+
if (!enabled) return text;
|
|
981
|
+
return `${COLORS[color]}${text}${COLORS.reset}`;
|
|
982
|
+
}
|
|
983
|
+
function severityColor(sev) {
|
|
984
|
+
if (sev === "error") return "red";
|
|
985
|
+
if (sev === "warning") return "yellow";
|
|
986
|
+
return "blue";
|
|
987
|
+
}
|
|
988
|
+
function severityLabel(sev) {
|
|
989
|
+
if (sev === "error") return "error";
|
|
990
|
+
if (sev === "warning") return "warn ";
|
|
991
|
+
return "info ";
|
|
992
|
+
}
|
|
993
|
+
function scoreColor(score) {
|
|
994
|
+
if (score >= 80) return "green";
|
|
995
|
+
if (score >= 50) return "yellow";
|
|
996
|
+
return "red";
|
|
997
|
+
}
|
|
998
|
+
function formatText(report, options = {}) {
|
|
999
|
+
const color = options.color ?? true;
|
|
1000
|
+
const out = [];
|
|
1001
|
+
const overallRounded = Math.round(report.overall);
|
|
1002
|
+
const bar = buildBar(overallRounded, 30);
|
|
1003
|
+
out.push(c("bold", `PromptScore \u2014 profile: ${report.profileName}`, color));
|
|
1004
|
+
out.push("");
|
|
1005
|
+
out.push(
|
|
1006
|
+
`${c("bold", "Overall", color)} ${c(scoreColor(overallRounded), `${overallRounded}/100`, color)} ${bar}`
|
|
1007
|
+
);
|
|
1008
|
+
out.push(c("dim", report.summary, color));
|
|
1009
|
+
out.push("");
|
|
1010
|
+
out.push(c("bold", "Categories", color));
|
|
1011
|
+
for (const category of report.categories) {
|
|
1012
|
+
const s = Math.round(category.score);
|
|
1013
|
+
out.push(
|
|
1014
|
+
` ${category.category.padEnd(16)} ${c(scoreColor(s), `${String(s).padStart(3)}/100`, color)} ${c("dim", `(${category.rules.length} rules)`, color)}`
|
|
1015
|
+
);
|
|
1016
|
+
}
|
|
1017
|
+
out.push("");
|
|
1018
|
+
const failed = report.results.filter((r) => !r.passed);
|
|
1019
|
+
if (failed.length > 0) {
|
|
1020
|
+
out.push(c("bold", "Findings", color));
|
|
1021
|
+
for (const result of failed) {
|
|
1022
|
+
const label = c(severityColor(result.severity), severityLabel(result.severity), color);
|
|
1023
|
+
out.push(` ${label} ${c("bold", result.ruleId, color)} ${result.message}`);
|
|
1024
|
+
if (result.suggestion) {
|
|
1025
|
+
out.push(c("dim", ` \u2192 ${result.suggestion}`, color));
|
|
1026
|
+
}
|
|
1027
|
+
if (result.reference) {
|
|
1028
|
+
out.push(c("dim", ` see: ${result.reference}`, color));
|
|
1029
|
+
}
|
|
1030
|
+
}
|
|
1031
|
+
out.push("");
|
|
1032
|
+
}
|
|
1033
|
+
const passed = report.results.filter((r) => r.passed);
|
|
1034
|
+
if (passed.length > 0) {
|
|
1035
|
+
out.push(c("dim", `${passed.length} rule${passed.length === 1 ? "" : "s"} passed.`, color));
|
|
1036
|
+
}
|
|
1037
|
+
return out.join("\n");
|
|
1038
|
+
}
|
|
1039
|
+
function buildBar(score, width) {
|
|
1040
|
+
const filled = Math.round(score / 100 * width);
|
|
1041
|
+
const empty = width - filled;
|
|
1042
|
+
return `[${"\u2588".repeat(filled)}${"\u2591".repeat(empty)}]`;
|
|
1043
|
+
}
|
|
1044
|
+
function formatJson(report, pretty = true) {
|
|
1045
|
+
return JSON.stringify(report, null, pretty ? 2 : 0);
|
|
1046
|
+
}
|
|
1047
|
+
function formatMarkdown(report) {
|
|
1048
|
+
const out = [];
|
|
1049
|
+
out.push(`# PromptScore \u2014 ${report.profileName}`);
|
|
1050
|
+
out.push("");
|
|
1051
|
+
out.push(`**Overall:** ${Math.round(report.overall)}/100`);
|
|
1052
|
+
out.push("");
|
|
1053
|
+
out.push(`> ${report.summary}`);
|
|
1054
|
+
out.push("");
|
|
1055
|
+
out.push("## Categories");
|
|
1056
|
+
out.push("");
|
|
1057
|
+
out.push("| Category | Score | Rules |");
|
|
1058
|
+
out.push("| --- | --- | --- |");
|
|
1059
|
+
for (const category of report.categories) {
|
|
1060
|
+
out.push(
|
|
1061
|
+
`| ${category.category} | ${Math.round(category.score)}/100 | ${category.rules.length} |`
|
|
1062
|
+
);
|
|
1063
|
+
}
|
|
1064
|
+
out.push("");
|
|
1065
|
+
const failed = report.results.filter((r) => !r.passed);
|
|
1066
|
+
if (failed.length > 0) {
|
|
1067
|
+
out.push("## Findings");
|
|
1068
|
+
out.push("");
|
|
1069
|
+
for (const result of failed) {
|
|
1070
|
+
out.push(`### \`${result.ruleId}\` \u2014 ${result.severity}`);
|
|
1071
|
+
out.push("");
|
|
1072
|
+
out.push(result.message);
|
|
1073
|
+
if (result.suggestion) {
|
|
1074
|
+
out.push("");
|
|
1075
|
+
out.push(`**Suggestion:** ${result.suggestion}`);
|
|
1076
|
+
}
|
|
1077
|
+
if (result.reference) {
|
|
1078
|
+
out.push("");
|
|
1079
|
+
out.push(`**Reference:** ${result.reference}`);
|
|
1080
|
+
}
|
|
1081
|
+
out.push("");
|
|
1082
|
+
}
|
|
1083
|
+
}
|
|
1084
|
+
return out.join("\n");
|
|
1085
|
+
}
|
|
1086
|
+
function format(report, fmt, options = {}) {
|
|
1087
|
+
if (fmt === "json") return formatJson(report);
|
|
1088
|
+
if (fmt === "markdown") return formatMarkdown(report);
|
|
1089
|
+
return formatText(report, options);
|
|
1090
|
+
}
|
|
1091
|
+
|
|
1092
|
+
// src/index.ts
|
|
1093
|
+
async function analyze(prompt, options = {}) {
|
|
1094
|
+
const ast = parsePrompt(prompt);
|
|
1095
|
+
const registry = options.registry ?? createDefaultRegistry();
|
|
1096
|
+
if (options.extraRules) {
|
|
1097
|
+
for (const rule of options.extraRules) {
|
|
1098
|
+
if (!registry.has(rule.id)) registry.register(rule);
|
|
1099
|
+
}
|
|
1100
|
+
}
|
|
1101
|
+
let profile;
|
|
1102
|
+
if (options.profile) {
|
|
1103
|
+
profile = options.profile;
|
|
1104
|
+
} else {
|
|
1105
|
+
const loader = new ProfileLoader(options.profileOptions ?? {});
|
|
1106
|
+
profile = await loader.load(options.model ?? "_base");
|
|
1107
|
+
}
|
|
1108
|
+
const results = await runRules({
|
|
1109
|
+
rules: registry.all(),
|
|
1110
|
+
profile,
|
|
1111
|
+
ast,
|
|
1112
|
+
only: options.only,
|
|
1113
|
+
includeLlm: options.includeLlm
|
|
1114
|
+
});
|
|
1115
|
+
return buildReport(results, profile);
|
|
1116
|
+
}
|
|
1117
|
+
|
|
1118
|
+
export { ProfileLoader, RuleRegistry, analyze, buildReport, builtinProfiles, createDefaultRegistry, deterministicRules, format, formatJson, formatMarkdown, formatText, getBuiltinProfile, parsePrompt, runRules };
|
|
1119
|
+
//# sourceMappingURL=index.js.map
|
|
1120
|
+
//# sourceMappingURL=index.js.map
|