@learning-commons/evaluators 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -0
- package/README.md +191 -45
- package/dist/base-DKcAYXfb.d.cts +464 -0
- package/dist/base-DKcAYXfb.d.ts +464 -0
- package/dist/batch/cli.js +4326 -0
- package/dist/batch/cli.js.map +1 -0
- package/dist/batch/index.cjs +3989 -0
- package/dist/batch/index.cjs.map +1 -0
- package/dist/batch/index.d.cts +146 -0
- package/dist/batch/index.d.ts +146 -0
- package/dist/batch/index.js +3958 -0
- package/dist/batch/index.js.map +1 -0
- package/dist/index.cjs +610 -213
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +153 -414
- package/dist/index.d.ts +153 -414
- package/dist/index.js +606 -214
- package/dist/index.js.map +1 -1
- package/package.json +35 -11
- package/src/batch/README.md +166 -0
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
+
import { B as BaseEvaluator, P as Provider, a as BaseEvaluatorConfig } from './base-DKcAYXfb.js';
|
|
3
|
+
export { E as EvaluatorMetadata, L as LLMProvider, b as LLMRequest, c as LLMResponse, d as LogContext, e as LogLevel, f as Logger, M as Message, g as ModelOverride, h as ProviderConfig, i as Providers, T as TelemetryOptions, j as TextGenerationResponse } from './base-DKcAYXfb.js';
|
|
2
4
|
|
|
3
5
|
/**
|
|
4
6
|
* Shared complexity levels used across all text complexity evaluators
|
|
@@ -60,6 +62,111 @@ declare const GradeLevelAppropriatenessSchema: z.ZodObject<{
|
|
|
60
62
|
}>;
|
|
61
63
|
type GradeLevelAppropriatenessInternal = z.infer<typeof GradeLevelAppropriatenessSchema>;
|
|
62
64
|
|
|
65
|
+
declare const PurposeOutputSchema: z.ZodObject<{
|
|
66
|
+
complexity_score: z.ZodEnum<["slightly_complex", "moderately_complex", "very_complex", "exceedingly_complex", "more_context_needed"]>;
|
|
67
|
+
reasoning: z.ZodString;
|
|
68
|
+
details: z.ZodObject<{
|
|
69
|
+
detailed_summary: z.ZodArray<z.ZodObject<{
|
|
70
|
+
factor: z.ZodString;
|
|
71
|
+
description: z.ZodString;
|
|
72
|
+
effect_on_complexity_dimension: z.ZodString;
|
|
73
|
+
}, "strict", z.ZodTypeAny, {
|
|
74
|
+
factor: string;
|
|
75
|
+
description: string;
|
|
76
|
+
effect_on_complexity_dimension: string;
|
|
77
|
+
}, {
|
|
78
|
+
factor: string;
|
|
79
|
+
description: string;
|
|
80
|
+
effect_on_complexity_dimension: string;
|
|
81
|
+
}>, "many">;
|
|
82
|
+
adjustment_and_scaffolding: z.ZodArray<z.ZodObject<{
|
|
83
|
+
scaffolding_need: z.ZodString;
|
|
84
|
+
suggestion: z.ZodString;
|
|
85
|
+
}, "strict", z.ZodTypeAny, {
|
|
86
|
+
scaffolding_need: string;
|
|
87
|
+
suggestion: string;
|
|
88
|
+
}, {
|
|
89
|
+
scaffolding_need: string;
|
|
90
|
+
suggestion: string;
|
|
91
|
+
}>, "many">;
|
|
92
|
+
recommended_use_cases: z.ZodArray<z.ZodObject<{
|
|
93
|
+
opportunity: z.ZodString;
|
|
94
|
+
suggestion: z.ZodString;
|
|
95
|
+
}, "strict", z.ZodTypeAny, {
|
|
96
|
+
suggestion: string;
|
|
97
|
+
opportunity: string;
|
|
98
|
+
}, {
|
|
99
|
+
suggestion: string;
|
|
100
|
+
opportunity: string;
|
|
101
|
+
}>, "many">;
|
|
102
|
+
}, "strict", z.ZodTypeAny, {
|
|
103
|
+
detailed_summary: {
|
|
104
|
+
factor: string;
|
|
105
|
+
description: string;
|
|
106
|
+
effect_on_complexity_dimension: string;
|
|
107
|
+
}[];
|
|
108
|
+
adjustment_and_scaffolding: {
|
|
109
|
+
scaffolding_need: string;
|
|
110
|
+
suggestion: string;
|
|
111
|
+
}[];
|
|
112
|
+
recommended_use_cases: {
|
|
113
|
+
suggestion: string;
|
|
114
|
+
opportunity: string;
|
|
115
|
+
}[];
|
|
116
|
+
}, {
|
|
117
|
+
detailed_summary: {
|
|
118
|
+
factor: string;
|
|
119
|
+
description: string;
|
|
120
|
+
effect_on_complexity_dimension: string;
|
|
121
|
+
}[];
|
|
122
|
+
adjustment_and_scaffolding: {
|
|
123
|
+
scaffolding_need: string;
|
|
124
|
+
suggestion: string;
|
|
125
|
+
}[];
|
|
126
|
+
recommended_use_cases: {
|
|
127
|
+
suggestion: string;
|
|
128
|
+
opportunity: string;
|
|
129
|
+
}[];
|
|
130
|
+
}>;
|
|
131
|
+
}, "strict", z.ZodTypeAny, {
|
|
132
|
+
reasoning: string;
|
|
133
|
+
complexity_score: "slightly_complex" | "moderately_complex" | "very_complex" | "exceedingly_complex" | "more_context_needed";
|
|
134
|
+
details: {
|
|
135
|
+
detailed_summary: {
|
|
136
|
+
factor: string;
|
|
137
|
+
description: string;
|
|
138
|
+
effect_on_complexity_dimension: string;
|
|
139
|
+
}[];
|
|
140
|
+
adjustment_and_scaffolding: {
|
|
141
|
+
scaffolding_need: string;
|
|
142
|
+
suggestion: string;
|
|
143
|
+
}[];
|
|
144
|
+
recommended_use_cases: {
|
|
145
|
+
suggestion: string;
|
|
146
|
+
opportunity: string;
|
|
147
|
+
}[];
|
|
148
|
+
};
|
|
149
|
+
}, {
|
|
150
|
+
reasoning: string;
|
|
151
|
+
complexity_score: "slightly_complex" | "moderately_complex" | "very_complex" | "exceedingly_complex" | "more_context_needed";
|
|
152
|
+
details: {
|
|
153
|
+
detailed_summary: {
|
|
154
|
+
factor: string;
|
|
155
|
+
description: string;
|
|
156
|
+
effect_on_complexity_dimension: string;
|
|
157
|
+
}[];
|
|
158
|
+
adjustment_and_scaffolding: {
|
|
159
|
+
scaffolding_need: string;
|
|
160
|
+
suggestion: string;
|
|
161
|
+
}[];
|
|
162
|
+
recommended_use_cases: {
|
|
163
|
+
suggestion: string;
|
|
164
|
+
opportunity: string;
|
|
165
|
+
}[];
|
|
166
|
+
};
|
|
167
|
+
}>;
|
|
168
|
+
type PurposeInternal = z.infer<typeof PurposeOutputSchema>;
|
|
169
|
+
|
|
63
170
|
/**
|
|
64
171
|
* Custom error types for the Evaluators SDK
|
|
65
172
|
*
|
|
@@ -202,151 +309,6 @@ declare class TimeoutError extends APIError {
|
|
|
202
309
|
constructor(message?: string);
|
|
203
310
|
}
|
|
204
311
|
|
|
205
|
-
/**
|
|
206
|
-
* Logging interface for the Evaluators SDK
|
|
207
|
-
*
|
|
208
|
-
* Provides structured logging with verbosity levels.
|
|
209
|
-
* Users can inject custom loggers or use the default console logger.
|
|
210
|
-
*/
|
|
211
|
-
/**
|
|
212
|
-
* Log levels in order of verbosity
|
|
213
|
-
*/
|
|
214
|
-
declare enum LogLevel {
|
|
215
|
-
/** Debug messages - very verbose, for development */
|
|
216
|
-
DEBUG = 0,
|
|
217
|
-
/** Informational messages - normal operations */
|
|
218
|
-
INFO = 1,
|
|
219
|
-
/** Warning messages - potentially problematic situations */
|
|
220
|
-
WARN = 2,
|
|
221
|
-
/** Error messages - errors that need attention */
|
|
222
|
-
ERROR = 3,
|
|
223
|
-
/** Silent - no logging */
|
|
224
|
-
SILENT = 4
|
|
225
|
-
}
|
|
226
|
-
/**
|
|
227
|
-
* Context object for structured logging
|
|
228
|
-
*/
|
|
229
|
-
interface LogContext {
|
|
230
|
-
/** Evaluator type (vocabulary, sentence-structure, etc.) */
|
|
231
|
-
evaluator?: string;
|
|
232
|
-
/** Current operation or stage */
|
|
233
|
-
operation?: string;
|
|
234
|
-
/** Error object if applicable */
|
|
235
|
-
error?: Error;
|
|
236
|
-
/** Additional metadata */
|
|
237
|
-
[key: string]: unknown;
|
|
238
|
-
}
|
|
239
|
-
/**
|
|
240
|
-
* Logger interface
|
|
241
|
-
*
|
|
242
|
-
* Implement this interface to provide custom logging behavior.
|
|
243
|
-
*
|
|
244
|
-
* @example
|
|
245
|
-
* ```typescript
|
|
246
|
-
* const customLogger: Logger = {
|
|
247
|
-
* debug: (msg, ctx) => myLogger.debug(msg, ctx),
|
|
248
|
-
* info: (msg, ctx) => myLogger.info(msg, ctx),
|
|
249
|
-
* warn: (msg, ctx) => myLogger.warn(msg, ctx),
|
|
250
|
-
* error: (msg, ctx) => myLogger.error(msg, ctx),
|
|
251
|
-
* };
|
|
252
|
-
*
|
|
253
|
-
* const evaluator = new VocabularyEvaluator({
|
|
254
|
-
* googleApiKey: '...',
|
|
255
|
-
* openaiApiKey: '...',
|
|
256
|
-
* logger: customLogger,
|
|
257
|
-
* logLevel: LogLevel.INFO,
|
|
258
|
-
* });
|
|
259
|
-
* ```
|
|
260
|
-
*/
|
|
261
|
-
interface Logger {
|
|
262
|
-
/**
|
|
263
|
-
* Log debug message
|
|
264
|
-
* Used for detailed debugging information
|
|
265
|
-
*/
|
|
266
|
-
debug(message: string, context?: LogContext): void;
|
|
267
|
-
/**
|
|
268
|
-
* Log informational message
|
|
269
|
-
* Used for normal operations
|
|
270
|
-
*/
|
|
271
|
-
info(message: string, context?: LogContext): void;
|
|
272
|
-
/**
|
|
273
|
-
* Log warning message
|
|
274
|
-
* Used for potentially problematic situations
|
|
275
|
-
*/
|
|
276
|
-
warn(message: string, context?: LogContext): void;
|
|
277
|
-
/**
|
|
278
|
-
* Log error message
|
|
279
|
-
* Used for errors that need attention
|
|
280
|
-
*/
|
|
281
|
-
error(message: string, context?: LogContext): void;
|
|
282
|
-
}
|
|
283
|
-
|
|
284
|
-
/**
|
|
285
|
-
* Message format for LLM conversations
|
|
286
|
-
*/
|
|
287
|
-
interface Message {
|
|
288
|
-
role: 'system' | 'user' | 'assistant';
|
|
289
|
-
content: string;
|
|
290
|
-
}
|
|
291
|
-
/**
|
|
292
|
-
* Request configuration for structured LLM generation
|
|
293
|
-
*/
|
|
294
|
-
interface LLMRequest<T> {
|
|
295
|
-
messages: Message[];
|
|
296
|
-
schema: z.ZodSchema<T>;
|
|
297
|
-
temperature?: number;
|
|
298
|
-
maxTokens?: number;
|
|
299
|
-
model?: string;
|
|
300
|
-
}
|
|
301
|
-
/**
|
|
302
|
-
* Response from LLM with usage metadata
|
|
303
|
-
*/
|
|
304
|
-
interface LLMResponse<T> {
|
|
305
|
-
data: T;
|
|
306
|
-
model: string;
|
|
307
|
-
usage: {
|
|
308
|
-
inputTokens: number;
|
|
309
|
-
outputTokens: number;
|
|
310
|
-
};
|
|
311
|
-
latencyMs: number;
|
|
312
|
-
}
|
|
313
|
-
/**
|
|
314
|
-
* Response from plain text generation
|
|
315
|
-
*/
|
|
316
|
-
interface TextGenerationResponse {
|
|
317
|
-
text: string;
|
|
318
|
-
usage: {
|
|
319
|
-
inputTokens: number;
|
|
320
|
-
outputTokens: number;
|
|
321
|
-
};
|
|
322
|
-
latencyMs: number;
|
|
323
|
-
}
|
|
324
|
-
/**
|
|
325
|
-
* Base interface for LLM provider implementations
|
|
326
|
-
*/
|
|
327
|
-
interface LLMProvider {
|
|
328
|
-
/**
|
|
329
|
-
* Generate structured output from LLM using Zod schema
|
|
330
|
-
*/
|
|
331
|
-
generateStructured<T>(request: LLMRequest<T>): Promise<LLMResponse<T>>;
|
|
332
|
-
/**
|
|
333
|
-
* Generate plain text from LLM
|
|
334
|
-
*/
|
|
335
|
-
generateText(messages: Message[], temperature?: number): Promise<TextGenerationResponse>;
|
|
336
|
-
}
|
|
337
|
-
/**
|
|
338
|
-
* Configuration for LLM provider
|
|
339
|
-
*/
|
|
340
|
-
interface ProviderConfig {
|
|
341
|
-
type: 'openai' | 'anthropic' | 'google' | 'custom';
|
|
342
|
-
apiKey?: string;
|
|
343
|
-
model?: string;
|
|
344
|
-
temperature?: number;
|
|
345
|
-
baseURL?: string;
|
|
346
|
-
customProvider?: LLMProvider;
|
|
347
|
-
maxRetries?: number;
|
|
348
|
-
}
|
|
349
|
-
|
|
350
312
|
/**
|
|
351
313
|
* Stage 1: Detailed sentence analysis output (40+ metrics)
|
|
352
314
|
* Ported from Python SentenceAnalysesEvaluatorOutput
|
|
@@ -538,18 +500,18 @@ declare const VocabularyComplexitySchema: z.ZodObject<{
|
|
|
538
500
|
reasoning: z.ZodString;
|
|
539
501
|
}, "strip", z.ZodTypeAny, {
|
|
540
502
|
reasoning: string;
|
|
503
|
+
complexity_score: "Slightly complex" | "Moderately complex" | "Very complex" | "Exceedingly complex";
|
|
541
504
|
tier_2_words: string;
|
|
542
505
|
tier_3_words: string;
|
|
543
506
|
archaic_words: string;
|
|
544
507
|
other_complex_words: string;
|
|
545
|
-
complexity_score: "Slightly complex" | "Moderately complex" | "Very complex" | "Exceedingly complex";
|
|
546
508
|
}, {
|
|
547
509
|
reasoning: string;
|
|
510
|
+
complexity_score: "Slightly complex" | "Moderately complex" | "Very complex" | "Exceedingly complex";
|
|
548
511
|
tier_2_words: string;
|
|
549
512
|
tier_3_words: string;
|
|
550
513
|
archaic_words: string;
|
|
551
514
|
other_complex_words: string;
|
|
552
|
-
complexity_score: "Slightly complex" | "Moderately complex" | "Very complex" | "Exceedingly complex";
|
|
553
515
|
}>;
|
|
554
516
|
type VocabularyInternal = z.infer<typeof VocabularyComplexitySchema>;
|
|
555
517
|
|
|
@@ -604,257 +566,6 @@ declare const ConventionalityOutputSchema: z.ZodObject<{
|
|
|
604
566
|
}>;
|
|
605
567
|
type ConventionalityInternal = z.infer<typeof ConventionalityOutputSchema>;
|
|
606
568
|
|
|
607
|
-
/**
|
|
608
|
-
* Evaluation status
|
|
609
|
-
*/
|
|
610
|
-
type EvaluationStatus = 'success' | 'error';
|
|
611
|
-
/**
|
|
612
|
-
* Token usage metrics from LLM providers
|
|
613
|
-
*/
|
|
614
|
-
interface TokenUsage {
|
|
615
|
-
input_tokens: number;
|
|
616
|
-
output_tokens: number;
|
|
617
|
-
}
|
|
618
|
-
/**
|
|
619
|
-
* Per-stage details for multi-stage evaluations
|
|
620
|
-
*/
|
|
621
|
-
interface StageDetail {
|
|
622
|
-
/** Stage name (e.g., "background_knowledge", "complexity_evaluation") */
|
|
623
|
-
stage: string;
|
|
624
|
-
/** Provider used for this stage (e.g., "openai:gpt-4o") */
|
|
625
|
-
provider: string;
|
|
626
|
-
/** Total latency including all retries (ms) */
|
|
627
|
-
latency_ms: number;
|
|
628
|
-
/** Token usage aggregated across all attempts */
|
|
629
|
-
token_usage?: TokenUsage;
|
|
630
|
-
/**
|
|
631
|
-
* Whether schema validation failed (indicates prompt needs clearer instructions)
|
|
632
|
-
*
|
|
633
|
-
* TODO: Not currently tracked. Vercel AI SDK abstracts validation away.
|
|
634
|
-
* To implement: Add custom retry wrapper that catches validation errors.
|
|
635
|
-
*/
|
|
636
|
-
schema_validation_failed?: boolean;
|
|
637
|
-
}
|
|
638
|
-
/**
|
|
639
|
-
* Extensible metadata for telemetry events
|
|
640
|
-
*/
|
|
641
|
-
interface TelemetryMetadata {
|
|
642
|
-
/** Detailed breakdown by stage (for multi-stage evaluations) */
|
|
643
|
-
stage_details?: StageDetail[];
|
|
644
|
-
}
|
|
645
|
-
/**
|
|
646
|
-
* Telemetry event payload
|
|
647
|
-
*/
|
|
648
|
-
interface TelemetryEvent {
|
|
649
|
-
timestamp: string;
|
|
650
|
-
sdk_version: string;
|
|
651
|
-
evaluator_type: string;
|
|
652
|
-
grade?: string;
|
|
653
|
-
status: EvaluationStatus;
|
|
654
|
-
error_code?: string;
|
|
655
|
-
latency_ms: number;
|
|
656
|
-
text_length_chars: number;
|
|
657
|
-
provider: string;
|
|
658
|
-
token_usage?: TokenUsage;
|
|
659
|
-
metadata?: TelemetryMetadata;
|
|
660
|
-
input_text?: string;
|
|
661
|
-
}
|
|
662
|
-
/**
|
|
663
|
-
* Configuration for telemetry client
|
|
664
|
-
*/
|
|
665
|
-
interface TelemetryConfig {
|
|
666
|
-
/** Analytics service endpoint URL */
|
|
667
|
-
endpoint: string;
|
|
668
|
-
/** Learning Commons partner key (optional, sent as X-API-Key header) */
|
|
669
|
-
partnerKey?: string;
|
|
670
|
-
/** Client ID for anonymous tracking (persistent UUID from ~/.config/learning-commons/config.json) */
|
|
671
|
-
clientId: string;
|
|
672
|
-
/** Enable telemetry (default: true) */
|
|
673
|
-
enabled: boolean;
|
|
674
|
-
/** Logger instance (respects the SDK's configured log level and custom logger) */
|
|
675
|
-
logger: Logger;
|
|
676
|
-
}
|
|
677
|
-
|
|
678
|
-
/**
|
|
679
|
-
* Telemetry client for sending analytics events
|
|
680
|
-
*
|
|
681
|
-
* Fire-and-forget implementation that never blocks SDK operations.
|
|
682
|
-
* Errors are logged but don't fail evaluations.
|
|
683
|
-
*/
|
|
684
|
-
declare class TelemetryClient {
|
|
685
|
-
private config;
|
|
686
|
-
private logger;
|
|
687
|
-
constructor(config: TelemetryConfig);
|
|
688
|
-
/**
|
|
689
|
-
* Send telemetry event to analytics service
|
|
690
|
-
*
|
|
691
|
-
* Fire-and-forget: Errors are logged but don't throw.
|
|
692
|
-
*/
|
|
693
|
-
send(event: TelemetryEvent): Promise<void>;
|
|
694
|
-
}
|
|
695
|
-
|
|
696
|
-
/**
|
|
697
|
-
* Granular telemetry configuration options
|
|
698
|
-
*/
|
|
699
|
-
interface TelemetryOptions {
|
|
700
|
-
/** Enable telemetry (default: true) */
|
|
701
|
-
enabled?: boolean;
|
|
702
|
-
/** Record input text in telemetry (default: false) */
|
|
703
|
-
recordInputs?: boolean;
|
|
704
|
-
}
|
|
705
|
-
/**
|
|
706
|
-
* Base configuration for all evaluators
|
|
707
|
-
*/
|
|
708
|
-
interface BaseEvaluatorConfig {
|
|
709
|
-
/** Google API key (for evaluators using Gemini) */
|
|
710
|
-
googleApiKey?: string;
|
|
711
|
-
/** OpenAI API key (for evaluators using GPT) */
|
|
712
|
-
openaiApiKey?: string;
|
|
713
|
-
/** Learning Commons partner key for authenticated telemetry (optional) */
|
|
714
|
-
partnerKey?: string;
|
|
715
|
-
/**
|
|
716
|
-
* Maximum number of retries for failed API calls (default: 2)
|
|
717
|
-
* Set to 0 to disable retries.
|
|
718
|
-
*
|
|
719
|
-
* Note: With maxRetries=2, a failed call will be attempted up to 3 times total
|
|
720
|
-
* (1 initial attempt + 2 retries)
|
|
721
|
-
*/
|
|
722
|
-
maxRetries?: number;
|
|
723
|
-
/**
|
|
724
|
-
* Telemetry configuration (default: all enabled)
|
|
725
|
-
*
|
|
726
|
-
* Can be:
|
|
727
|
-
* - `true`: Enable with defaults (recordInputs: false)
|
|
728
|
-
* - `false`: Disable completely
|
|
729
|
-
* - `TelemetryOptions`: Granular control
|
|
730
|
-
*/
|
|
731
|
-
telemetry?: boolean | TelemetryOptions;
|
|
732
|
-
/**
|
|
733
|
-
* Custom logger implementation (optional)
|
|
734
|
-
* If not provided, uses console logger with specified logLevel
|
|
735
|
-
*/
|
|
736
|
-
logger?: Logger;
|
|
737
|
-
/**
|
|
738
|
-
* Log level for default console logger (default: WARN)
|
|
739
|
-
* Only used if custom logger is not provided
|
|
740
|
-
*
|
|
741
|
-
* - DEBUG: Very verbose, shows all operations
|
|
742
|
-
* - INFO: Normal operations
|
|
743
|
-
* - WARN: Warnings only (default)
|
|
744
|
-
* - ERROR: Errors only
|
|
745
|
-
* - SILENT: No logging
|
|
746
|
-
*/
|
|
747
|
-
logLevel?: LogLevel;
|
|
748
|
-
}
|
|
749
|
-
/**
|
|
750
|
-
* Evaluator metadata interface
|
|
751
|
-
* Each evaluator must provide this metadata as static properties
|
|
752
|
-
*/
|
|
753
|
-
interface EvaluatorMetadata {
|
|
754
|
-
/** Unique identifier for the evaluator (e.g., 'vocabulary', 'sentence-structure') */
|
|
755
|
-
readonly id: string;
|
|
756
|
-
/** Human-readable name (e.g., 'Vocabulary', 'Sentence Structure') */
|
|
757
|
-
readonly name: string;
|
|
758
|
-
/** Brief description of what the evaluator does */
|
|
759
|
-
readonly description: string;
|
|
760
|
-
/** Supported grade levels (e.g., ['3', '4', '5', ...]) */
|
|
761
|
-
readonly supportedGrades: readonly string[];
|
|
762
|
-
/** Whether this evaluator requires a Google API key */
|
|
763
|
-
readonly requiresGoogleKey: boolean;
|
|
764
|
-
/** Whether this evaluator requires an OpenAI API key */
|
|
765
|
-
readonly requiresOpenAIKey: boolean;
|
|
766
|
-
}
|
|
767
|
-
/**
|
|
768
|
-
* Abstract base class for all evaluators
|
|
769
|
-
*
|
|
770
|
-
* Provides common functionality:
|
|
771
|
-
* - Telemetry setup and event sending
|
|
772
|
-
* - Text validation
|
|
773
|
-
* - Grade validation (with overridable default)
|
|
774
|
-
* - Metadata creation
|
|
775
|
-
*
|
|
776
|
-
* Concrete evaluators must implement:
|
|
777
|
-
* - static metadata: Provide evaluator metadata (see EvaluatorMetadata interface)
|
|
778
|
-
*/
|
|
779
|
-
declare abstract class BaseEvaluator {
|
|
780
|
-
protected telemetryClient?: TelemetryClient;
|
|
781
|
-
protected logger: Logger;
|
|
782
|
-
protected config: Required<Pick<BaseEvaluatorConfig, 'maxRetries'>> & {
|
|
783
|
-
telemetry: Required<TelemetryOptions>;
|
|
784
|
-
};
|
|
785
|
-
/**
|
|
786
|
-
* Static metadata for the evaluator
|
|
787
|
-
*
|
|
788
|
-
* Concrete evaluators MUST define this property.
|
|
789
|
-
*
|
|
790
|
-
* @example
|
|
791
|
-
* ```typescript
|
|
792
|
-
* class MyEvaluator extends BaseEvaluator {
|
|
793
|
-
* static readonly metadata = {
|
|
794
|
-
* id: 'my-evaluator',
|
|
795
|
-
* name: 'My Evaluator',
|
|
796
|
-
* description: 'Does something useful',
|
|
797
|
-
* supportedGrades: ['3', '4', '5'],
|
|
798
|
-
* requiresGoogleKey: true,
|
|
799
|
-
* requiresOpenAIKey: false,
|
|
800
|
-
* };
|
|
801
|
-
* }
|
|
802
|
-
* ```
|
|
803
|
-
*/
|
|
804
|
-
static readonly metadata: EvaluatorMetadata;
|
|
805
|
-
constructor(config: BaseEvaluatorConfig);
|
|
806
|
-
/**
|
|
807
|
-
* Get metadata for this evaluator instance
|
|
808
|
-
* @throws {ConfigurationError} If the subclass has not defined static metadata
|
|
809
|
-
*/
|
|
810
|
-
protected get metadata(): EvaluatorMetadata;
|
|
811
|
-
/**
|
|
812
|
-
* Validate that required API keys are provided based on metadata
|
|
813
|
-
* @throws {ConfigurationError} If required API keys are missing
|
|
814
|
-
*/
|
|
815
|
-
private validateApiKeys;
|
|
816
|
-
/**
|
|
817
|
-
* Normalize telemetry config to standard format
|
|
818
|
-
*/
|
|
819
|
-
private normalizeTelemetryConfig;
|
|
820
|
-
/**
|
|
821
|
-
* Get the evaluator type identifier from metadata
|
|
822
|
-
* @returns The evaluator type ID (e.g., "vocabulary", "sentence-structure")
|
|
823
|
-
*/
|
|
824
|
-
protected getEvaluatorType(): string;
|
|
825
|
-
/**
|
|
826
|
-
* Validate text meets requirements
|
|
827
|
-
* Default implementation - can be overridden by concrete evaluators
|
|
828
|
-
*
|
|
829
|
-
* @throws {ValidationError} If text is invalid
|
|
830
|
-
*/
|
|
831
|
-
protected validateText(text: string): void;
|
|
832
|
-
/**
|
|
833
|
-
* Validate grade is in supported range
|
|
834
|
-
* Default implementation - can be overridden by concrete evaluators
|
|
835
|
-
*
|
|
836
|
-
* @param grade - Grade level to validate
|
|
837
|
-
* @param validGrades - Set of valid grades for this evaluator
|
|
838
|
-
* @throws {ValidationError} If grade is invalid
|
|
839
|
-
*/
|
|
840
|
-
protected validateGrade(grade: string, validGrades: Set<string>): void;
|
|
841
|
-
/**
|
|
842
|
-
* Send telemetry event to analytics service
|
|
843
|
-
* Common helper for all evaluators
|
|
844
|
-
*/
|
|
845
|
-
protected sendTelemetry(params: {
|
|
846
|
-
status: 'success' | 'error';
|
|
847
|
-
latencyMs: number;
|
|
848
|
-
textLength: number;
|
|
849
|
-
grade?: string;
|
|
850
|
-
provider: string;
|
|
851
|
-
errorCode?: string;
|
|
852
|
-
tokenUsage?: TokenUsage;
|
|
853
|
-
metadata?: TelemetryMetadata;
|
|
854
|
-
inputText?: string;
|
|
855
|
-
}): Promise<void>;
|
|
856
|
-
}
|
|
857
|
-
|
|
858
569
|
/**
|
|
859
570
|
* Vocabulary Evaluator
|
|
860
571
|
*
|
|
@@ -887,8 +598,7 @@ declare class VocabularyEvaluator extends BaseEvaluator {
|
|
|
887
598
|
name: string;
|
|
888
599
|
description: string;
|
|
889
600
|
supportedGrades: readonly ["3", "4", "5", "6", "7", "8", "9", "10", "11", "12"];
|
|
890
|
-
|
|
891
|
-
requiresOpenAIKey: boolean;
|
|
601
|
+
defaultProviders: readonly [Provider.Google, Provider.OpenAI];
|
|
892
602
|
};
|
|
893
603
|
private grades34ComplexityProvider;
|
|
894
604
|
private otherGradesComplexityProvider;
|
|
@@ -901,6 +611,7 @@ declare class VocabularyEvaluator extends BaseEvaluator {
|
|
|
901
611
|
* @param grade - The target grade level (3-12)
|
|
902
612
|
* @returns Evaluation result with complexity score and detailed analysis
|
|
903
613
|
* @throws {ValidationError} If text is empty, too short/long, or grade is invalid
|
|
614
|
+
* @throws {ConfigurationError} If modelOverride specifies a model ID that the provider rejects
|
|
904
615
|
* @throws {APIError} If LLM API calls fail (includes AuthenticationError, RateLimitError, NetworkError, TimeoutError)
|
|
905
616
|
*/
|
|
906
617
|
evaluate(text: string, grade: string): Promise<EvaluationResult<TextComplexityLevel, VocabularyInternal>>;
|
|
@@ -967,11 +678,9 @@ declare class SentenceStructureEvaluator extends BaseEvaluator {
|
|
|
967
678
|
name: string;
|
|
968
679
|
description: string;
|
|
969
680
|
supportedGrades: readonly ["3", "4", "5", "6", "7", "8", "9", "10", "11", "12"];
|
|
970
|
-
|
|
971
|
-
requiresOpenAIKey: boolean;
|
|
681
|
+
defaultProviders: readonly [Provider.OpenAI];
|
|
972
682
|
};
|
|
973
|
-
private
|
|
974
|
-
private complexityProvider;
|
|
683
|
+
private provider;
|
|
975
684
|
constructor(config: BaseEvaluatorConfig);
|
|
976
685
|
/**
|
|
977
686
|
* Evaluate sentence structure complexity for a given text and grade level
|
|
@@ -980,6 +689,7 @@ declare class SentenceStructureEvaluator extends BaseEvaluator {
|
|
|
980
689
|
* @param grade - The target grade level (3-12)
|
|
981
690
|
* @returns Evaluation result with complexity score and detailed analysis
|
|
982
691
|
* @throws {ValidationError} If text is empty, too short/long, or grade is invalid
|
|
692
|
+
* @throws {ConfigurationError} If modelOverride specifies a model ID that the provider rejects
|
|
983
693
|
* @throws {APIError} If LLM API calls fail (includes AuthenticationError, RateLimitError, NetworkError, TimeoutError)
|
|
984
694
|
*/
|
|
985
695
|
evaluate(text: string, grade: string): Promise<EvaluationResult<TextComplexityLevel, SentenceStructureInternal>>;
|
|
@@ -1045,8 +755,7 @@ declare class GradeLevelAppropriatenessEvaluator extends BaseEvaluator {
|
|
|
1045
755
|
name: string;
|
|
1046
756
|
description: string;
|
|
1047
757
|
supportedGrades: readonly [];
|
|
1048
|
-
|
|
1049
|
-
requiresOpenAIKey: boolean;
|
|
758
|
+
defaultProviders: readonly [Provider.Google];
|
|
1050
759
|
};
|
|
1051
760
|
private provider;
|
|
1052
761
|
constructor(config: BaseEvaluatorConfig);
|
|
@@ -1056,6 +765,7 @@ declare class GradeLevelAppropriatenessEvaluator extends BaseEvaluator {
|
|
|
1056
765
|
* @param text - The text to evaluate
|
|
1057
766
|
* @returns Evaluation result with grade recommendations and scaffolding suggestions
|
|
1058
767
|
* @throws {ValidationError} If text is empty or too short/long
|
|
768
|
+
* @throws {ConfigurationError} If modelOverride specifies a model ID that the provider rejects
|
|
1059
769
|
* @throws {APIError} If LLM API calls fail (includes AuthenticationError, RateLimitError, NetworkError, TimeoutError)
|
|
1060
770
|
*/
|
|
1061
771
|
evaluate(text: string): Promise<EvaluationResult<GradeBand, GradeLevelAppropriatenessInternal>>;
|
|
@@ -1104,8 +814,7 @@ declare class SmkEvaluator extends BaseEvaluator {
|
|
|
1104
814
|
name: string;
|
|
1105
815
|
description: string;
|
|
1106
816
|
supportedGrades: readonly ["3", "4", "5", "6", "7", "8", "9", "10", "11", "12"];
|
|
1107
|
-
|
|
1108
|
-
requiresOpenAIKey: boolean;
|
|
817
|
+
defaultProviders: readonly [Provider.Google];
|
|
1109
818
|
};
|
|
1110
819
|
private provider;
|
|
1111
820
|
constructor(config: BaseEvaluatorConfig);
|
|
@@ -1116,6 +825,7 @@ declare class SmkEvaluator extends BaseEvaluator {
|
|
|
1116
825
|
* @param grade - The target grade level (3-12)
|
|
1117
826
|
* @returns Evaluation result with complexity score and detailed analysis
|
|
1118
827
|
* @throws {ValidationError} If text is empty, too short/long, or grade is invalid
|
|
828
|
+
* @throws {ConfigurationError} If modelOverride specifies a model ID that the provider rejects
|
|
1119
829
|
* @throws {APIError} If LLM API calls fail (includes AuthenticationError, RateLimitError, NetworkError, TimeoutError)
|
|
1120
830
|
*/
|
|
1121
831
|
evaluate(text: string, grade: string): Promise<EvaluationResult<TextComplexityLevel, SmkInternal>>;
|
|
@@ -1167,8 +877,7 @@ declare class ConventionalityEvaluator extends BaseEvaluator {
|
|
|
1167
877
|
name: string;
|
|
1168
878
|
description: string;
|
|
1169
879
|
supportedGrades: readonly ["3", "4", "5", "6", "7", "8", "9", "10", "11", "12"];
|
|
1170
|
-
|
|
1171
|
-
requiresOpenAIKey: boolean;
|
|
880
|
+
defaultProviders: readonly [Provider.Google];
|
|
1172
881
|
};
|
|
1173
882
|
private provider;
|
|
1174
883
|
constructor(config: BaseEvaluatorConfig);
|
|
@@ -1179,6 +888,7 @@ declare class ConventionalityEvaluator extends BaseEvaluator {
|
|
|
1179
888
|
* @param grade - The target grade level (3-12)
|
|
1180
889
|
* @returns Evaluation result with complexity score and detailed analysis
|
|
1181
890
|
* @throws {ValidationError} If text is empty, too short/long, or grade is invalid
|
|
891
|
+
* @throws {ConfigurationError} If modelOverride specifies a model ID that the provider rejects
|
|
1182
892
|
* @throws {APIError} If LLM API calls fail (includes AuthenticationError, RateLimitError, NetworkError, TimeoutError)
|
|
1183
893
|
*/
|
|
1184
894
|
evaluate(text: string, grade: string): Promise<EvaluationResult<TextComplexityLevel, ConventionalityInternal>>;
|
|
@@ -1250,8 +960,7 @@ declare class TextComplexityEvaluator extends BaseEvaluator {
|
|
|
1250
960
|
name: string;
|
|
1251
961
|
description: string;
|
|
1252
962
|
supportedGrades: readonly ["3", "4", "5", "6", "7", "8", "9", "10", "11", "12"];
|
|
1253
|
-
|
|
1254
|
-
requiresOpenAIKey: boolean;
|
|
963
|
+
defaultProviders: readonly [Provider.Google, Provider.OpenAI];
|
|
1255
964
|
};
|
|
1256
965
|
private vocabularyEvaluator;
|
|
1257
966
|
private sentenceStructureEvaluator;
|
|
@@ -1269,7 +978,8 @@ declare class TextComplexityEvaluator extends BaseEvaluator {
|
|
|
1269
978
|
* @param text - The text to evaluate
|
|
1270
979
|
* @param grade - The target grade level (3-12)
|
|
1271
980
|
* @returns Map of sub-evaluator results
|
|
1272
|
-
* @throws {ValidationError} If text is empty or grade is invalid
|
|
981
|
+
* @throws {ValidationError} If text is empty, too short/long, or grade is invalid
|
|
982
|
+
* @throws {ConfigurationError} If modelOverride specifies a model ID that the provider rejects
|
|
1273
983
|
* @throws {Error} If all sub-evaluators fail
|
|
1274
984
|
*/
|
|
1275
985
|
evaluate(text: string, grade: string): Promise<TextComplexityResult>;
|
|
@@ -1296,6 +1006,35 @@ declare class TextComplexityEvaluator extends BaseEvaluator {
|
|
|
1296
1006
|
*/
|
|
1297
1007
|
declare function evaluateTextComplexity(text: string, grade: string, config: BaseEvaluatorConfig): Promise<TextComplexityResult>;
|
|
1298
1008
|
|
|
1009
|
+
type PurposeComplexityLevel = TextComplexityLevel | 'More context needed';
|
|
1010
|
+
declare class PurposeEvaluator extends BaseEvaluator {
|
|
1011
|
+
static readonly metadata: {
|
|
1012
|
+
id: string;
|
|
1013
|
+
name: string;
|
|
1014
|
+
description: string;
|
|
1015
|
+
supportedGrades: string[];
|
|
1016
|
+
defaultProviders: readonly [Provider.Google];
|
|
1017
|
+
};
|
|
1018
|
+
private static readonly TEMPERATURE;
|
|
1019
|
+
private static computeFkScore;
|
|
1020
|
+
private provider;
|
|
1021
|
+
constructor(config: BaseEvaluatorConfig);
|
|
1022
|
+
/**
|
|
1023
|
+
* Evaluate purpose complexity for a given text and grade level
|
|
1024
|
+
*
|
|
1025
|
+
* @param text - The text to evaluate
|
|
1026
|
+
* @param grade - The target grade level (3-12)
|
|
1027
|
+
* @returns Evaluation result with complexity score and detailed analysis
|
|
1028
|
+
* @throws {ValidationError} If text is empty, too short/long, or grade is invalid
|
|
1029
|
+
* @throws {ConfigurationError} If modelOverride specifies a model ID that the provider rejects
|
|
1030
|
+
* @throws {APIError} If LLM API calls fail (includes AuthenticationError, RateLimitError, NetworkError, TimeoutError)
|
|
1031
|
+
*/
|
|
1032
|
+
evaluate(text: string, grade: string): Promise<EvaluationResult<PurposeComplexityLevel, PurposeInternal>>;
|
|
1033
|
+
private parseAndValidateGrade;
|
|
1034
|
+
private callLLM;
|
|
1035
|
+
}
|
|
1036
|
+
declare function evaluatePurpose(text: string, grade: string, config: BaseEvaluatorConfig): Promise<EvaluationResult<PurposeComplexityLevel, PurposeInternal>>;
|
|
1037
|
+
|
|
1299
1038
|
/**
|
|
1300
1039
|
* Calculate Flesch-Kincaid Grade Level
|
|
1301
1040
|
* Equivalent to Python's textstat.flesch_kincaid_grade()
|
|
@@ -1326,4 +1065,4 @@ declare function addEngineeredFeatures(analysis: SentenceAnalysis): SentenceFeat
|
|
|
1326
1065
|
*/
|
|
1327
1066
|
declare function featuresToJSON(features: SentenceFeatures, decimals?: number, castToInt?: boolean): string;
|
|
1328
1067
|
|
|
1329
|
-
export { APIError, AuthenticationError,
|
|
1068
|
+
export { APIError, AuthenticationError, BaseEvaluatorConfig, type ComplexityClassification, ComplexityClassificationSchema, ConfigurationError, ConventionalityEvaluator, type ConventionalityInternal, type EvaluationError, type EvaluationMetadata, type EvaluationResult, EvaluatorError, GradeBand, GradeLevelAppropriatenessEvaluator, type GradeLevelAppropriatenessInternal, GradeLevelAppropriatenessSchema, NetworkError, Provider, type PurposeComplexityLevel, PurposeEvaluator, type PurposeInternal, RateLimitError, type ReadabilityMetrics, type SentenceAnalysis, SentenceAnalysisSchema, type SentenceFeatures, SentenceStructureEvaluator, type SentenceStructureInternal, SmkEvaluator, type SmkInternal, TextComplexityEvaluator, TextComplexityLevel, type TextComplexityResult, TimeoutError, ValidationError, VocabularyEvaluator, type VocabularyInternal, addEngineeredFeatures, calculateFleschKincaidGrade, calculateReadabilityMetrics, evaluateConventionality, evaluateGradeLevelAppropriateness, evaluatePurpose, evaluateSentenceStructure, evaluateSmk, evaluateTextComplexity, evaluateVocabulary, featuresToJSON };
|