@lov3kaizen/agentsea-evaluate 0.5.1

package/dist/index-bPghFsfP.d.mts

@@ -0,0 +1,315 @@
+import { z } from 'zod';
+import { EventEmitter } from 'eventemitter3';
+
+type AnnotationTaskStatus = 'draft' | 'active' | 'paused' | 'completed' | 'cancelled';
+type AnnotationItemStatus = 'pending' | 'assigned' | 'in_progress' | 'completed' | 'flagged' | 'skipped';
+interface AnnotationTaskConfig {
+    name: string;
+    description: string;
+    instructions: string;
+    schema: z.ZodSchema;
+    itemsPerAnnotator?: number;
+    annotatorsPerItem?: number;
+    deadline?: Date;
+    allowSkip?: boolean;
+    requireExplanation?: boolean;
+    metadata?: Record<string, unknown>;
+}
+interface IAnnotationTask {
+    id: string;
+    name: string;
+    description: string;
+    instructions: string;
+    schema: z.ZodSchema;
+    status: AnnotationTaskStatus;
+    itemsPerAnnotator: number;
+    annotatorsPerItem: number;
+    deadline?: Date;
+    createdAt: number;
+    updatedAt: number;
+    completedAt?: number;
+    metadata?: Record<string, unknown>;
+}
+interface AnnotationItem {
+    id: string;
+    taskId: string;
+    data: Record<string, unknown>;
+    status: AnnotationItemStatus;
+    priority: number;
+    assignedTo?: string[];
+    annotations: Annotation[];
+    consensus?: ConsensusResult;
+    createdAt: number;
+    updatedAt: number;
+    metadata?: Record<string, unknown>;
+}
+interface Annotation {
+    id: string;
+    itemId: string;
+    annotatorId: string;
+    value: Record<string, unknown>;
+    explanation?: string;
+    confidence?: number;
+    duration: number;
+    createdAt: number;
+    isGold?: boolean;
+    quality?: AnnotationQuality;
+}
+interface AnnotationQuality {
+    score: number;
+    agreementWithConsensus?: number;
+    flagged?: boolean;
+    flagReason?: string;
+}
+interface Annotator {
+    id: string;
+    name: string;
+    email?: string;
+    role: 'annotator' | 'reviewer' | 'expert' | 'admin';
+    accuracy?: number;
+    completedCount: number;
+    activeTaskIds: string[];
+    metadata?: Record<string, unknown>;
+}
+interface AnnotatorStats {
+    annotatorId: string;
+    totalAnnotations: number;
+    averageDuration: number;
+    accuracy: number;
+    agreementRate: number;
+    flaggedRate: number;
+    byTask: Record<string, {
+        annotations: number;
+        accuracy: number;
+        avgDuration: number;
+    }>;
+}
+interface AnnotationQueueConfig {
+    task: IAnnotationTask;
+    items: AnnotationItem[];
+    prioritization?: PrioritizationType;
+    assignment?: AssignmentStrategy;
+    qualityControl?: QualityControlConfig;
+}
+type PrioritizationType = 'fifo' | 'uncertainty' | 'diversity' | 'importance' | 'custom';
+type AssignmentStrategy = 'round-robin' | 'expertise' | 'load-balanced' | 'random';
+interface QualityControlConfig {
+    goldStandard?: GoldStandardItem[];
+    goldRatio?: number;
+    agreementThreshold?: number;
+    minAnnotatorAccuracy?: number;
+    autoReject?: boolean;
+    expertReviewThreshold?: number;
+}
+interface GoldStandardItem {
+    itemId: string;
+    expectedAnnotation: Record<string, unknown>;
+    tolerance?: Record<string, number>;
+}
+type ConsensusMethod = 'majority' | 'unanimous' | 'weighted' | 'dawid-skene' | 'expert';
+interface ConsensusConfig {
+    method: ConsensusMethod;
+    minAgreement?: number;
+    weights?: Record<string, number>;
+    expertAnnotatorId?: string;
+    tieBreaker?: 'expert' | 'random' | 'none';
+}
+interface ConsensusResult {
+    value: Record<string, unknown>;
+    method: ConsensusMethod;
+    agreement: number;
+    confidence: number;
+    contributingAnnotations: string[];
+    disagreements?: Disagreement[];
+}
+interface Disagreement {
+    field: string;
+    values: Array<{
+        value: unknown;
+        annotatorIds: string[];
+        count: number;
+    }>;
+    resolved: boolean;
+    resolution?: unknown;
+}
+interface QueueStats {
+    taskId: string;
+    totalItems: number;
+    pendingItems: number;
+    assignedItems: number;
+    completedItems: number;
+    flaggedItems: number;
+    averageAnnotationsPerItem: number;
+    averageAgreement: number;
+    estimatedCompletion?: Date;
+}
+interface AnnotationResults {
+    taskId: string;
+    items: Array<{
+        itemId: string;
+        data: Record<string, unknown>;
+        consensus: ConsensusResult;
+        annotations: Annotation[];
+    }>;
+    stats: {
+        total: number;
+        completed: number;
+        agreementRate: number;
+        flaggedCount: number;
+        avgAnnotationsPerItem: number;
+    };
+}
+interface BatchAssignment {
+    annotatorId: string;
+    itemIds: string[];
+    deadline?: Date;
+    instructions?: string;
+}
+type AnnotationEventType = 'task:created' | 'task:started' | 'task:paused' | 'task:completed' | 'item:assigned' | 'item:annotated' | 'item:flagged' | 'item:consensus_reached' | 'annotator:joined' | 'annotator:warning' | 'quality:alert';
+interface AnnotationEvent {
+    type: AnnotationEventType;
+    taskId: string;
+    itemId?: string;
+    annotatorId?: string;
+    data?: Record<string, unknown>;
+    timestamp: number;
+}
+
+declare class AnnotationTask implements IAnnotationTask {
+    readonly id: string;
+    readonly name: string;
+    readonly description: string;
+    readonly instructions: string;
+    readonly schema: z.ZodSchema;
+    status: AnnotationTaskStatus;
+    readonly itemsPerAnnotator: number;
+    readonly annotatorsPerItem: number;
+    readonly deadline?: Date;
+    readonly createdAt: number;
+    updatedAt: number;
+    completedAt?: number;
+    readonly metadata?: Record<string, unknown>;
+    constructor(config: AnnotationTaskConfig);
+    start(): void;
+    pause(): void;
+    resume(): void;
+    complete(): void;
+    cancel(): void;
+    validateAnnotation(value: unknown): {
+        valid: boolean;
+        error?: string;
+    };
+    isPastDeadline(): boolean;
+    toConfig(): AnnotationTaskConfig;
+}
+declare function createAnnotationTask(config: AnnotationTaskConfig): AnnotationTask;
+declare const BinaryClassificationSchema: z.ZodObject<{
+    label: z.ZodEnum<["positive", "negative"]>;
+    confidence: z.ZodOptional<z.ZodNumber>;
+    notes: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    label: "positive" | "negative";
+    confidence?: number | undefined;
+    notes?: string | undefined;
+}, {
+    label: "positive" | "negative";
+    confidence?: number | undefined;
+    notes?: string | undefined;
+}>;
+declare const QualityRatingSchema: z.ZodObject<{
+    accuracy: z.ZodNumber;
+    helpfulness: z.ZodNumber;
+    safety: z.ZodEnum<["pass", "fail"]>;
+    corrections: z.ZodOptional<z.ZodString>;
+    notes: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    accuracy: number;
+    helpfulness: number;
+    safety: "pass" | "fail";
+    notes?: string | undefined;
+    corrections?: string | undefined;
+}, {
+    accuracy: number;
+    helpfulness: number;
+    safety: "pass" | "fail";
+    notes?: string | undefined;
+    corrections?: string | undefined;
+}>;
+declare const TextSpanSchema: z.ZodObject<{
+    spans: z.ZodArray<z.ZodObject<{
+        start: z.ZodNumber;
+        end: z.ZodNumber;
+        label: z.ZodString;
+        text: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        start: number;
+        label: string;
+        end: number;
+        text?: string | undefined;
+    }, {
+        start: number;
+        label: string;
+        end: number;
+        text?: string | undefined;
+    }>, "many">;
+    notes: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    spans: {
+        start: number;
+        label: string;
+        end: number;
+        text?: string | undefined;
+    }[];
+    notes?: string | undefined;
+}, {
+    spans: {
+        start: number;
+        label: string;
+        end: number;
+        text?: string | undefined;
+    }[];
+    notes?: string | undefined;
+}>;
+
+interface QueueEvents {
+    'item:assigned': (itemId: string, annotatorId: string) => void;
+    'item:annotated': (itemId: string, annotation: Annotation) => void;
+    'item:consensus': (itemId: string, consensus: ConsensusResult) => void;
+    'item:flagged': (itemId: string, reason: string) => void;
+}
+declare class AnnotationQueue extends EventEmitter<QueueEvents> {
+    readonly task: AnnotationTask;
+    private items;
+    private annotatorAssignments;
+    private annotatorCounts;
+    constructor(config: AnnotationQueueConfig);
+    getNextItem(annotatorId: string): AnnotationItem | null;
+    assignItem(itemId: string, annotatorId: string): void;
+    submitAnnotation(itemId: string, annotatorId: string, value: Record<string, unknown>, duration: number): Annotation;
+    flagItem(itemId: string, reason: string): void;
+    skipItem(itemId: string, annotatorId: string): void;
+    getBatchAssignment(annotatorId: string, count: number): BatchAssignment;
+    getStats(): QueueStats;
+    getItem(itemId: string): AnnotationItem | undefined;
+    getItems(): AnnotationItem[];
+    getItemsByStatus(status: AnnotationItemStatus): AnnotationItem[];
+    getAnnotatorCount(annotatorId: string): number;
+}
+declare function createAnnotationQueue(config: AnnotationQueueConfig): AnnotationQueue;
+
+declare class ConsensusManager {
+    private method;
+    private weights?;
+    private expertAnnotatorId?;
+    constructor(config: ConsensusConfig);
+    calculateConsensus(annotations: Annotation[]): ConsensusResult;
+    private majorityConsensus;
+    private findMajority;
+    private unanimousConsensus;
+    private weightedConsensus;
+    private expertConsensus;
+    calculateAgreement(annotations: Annotation[]): number;
+}
+declare function createConsensusManager(config: ConsensusConfig): ConsensusManager;
+
+export { type AnnotationTaskStatus as A, type BatchAssignment as B, type ConsensusMethod as C, type Disagreement as D, type GoldStandardItem as G, type IAnnotationTask as I, type PrioritizationType as P, type QualityControlConfig as Q, TextSpanSchema as T, type AnnotationItemStatus as a, type AnnotationTaskConfig as b, type AnnotationItem as c, type Annotation as d, type AnnotationQuality as e, type Annotator as f, type AnnotatorStats as g, type AnnotationQueueConfig as h, type AssignmentStrategy as i, type ConsensusConfig as j, type ConsensusResult as k, type QueueStats as l, type AnnotationResults as m, type AnnotationEventType as n, type AnnotationEvent as o, AnnotationTask as p, createAnnotationTask as q, BinaryClassificationSchema as r, QualityRatingSchema as s, AnnotationQueue as t, createAnnotationQueue as u, ConsensusManager as v, createConsensusManager as w };

package/dist/index-bPghFsfP.d.ts

@@ -0,0 +1,315 @@
+import { z } from 'zod';
+import { EventEmitter } from 'eventemitter3';
+
+type AnnotationTaskStatus = 'draft' | 'active' | 'paused' | 'completed' | 'cancelled';
+type AnnotationItemStatus = 'pending' | 'assigned' | 'in_progress' | 'completed' | 'flagged' | 'skipped';
+interface AnnotationTaskConfig {
+    name: string;
+    description: string;
+    instructions: string;
+    schema: z.ZodSchema;
+    itemsPerAnnotator?: number;
+    annotatorsPerItem?: number;
+    deadline?: Date;
+    allowSkip?: boolean;
+    requireExplanation?: boolean;
+    metadata?: Record<string, unknown>;
+}
+interface IAnnotationTask {
+    id: string;
+    name: string;
+    description: string;
+    instructions: string;
+    schema: z.ZodSchema;
+    status: AnnotationTaskStatus;
+    itemsPerAnnotator: number;
+    annotatorsPerItem: number;
+    deadline?: Date;
+    createdAt: number;
+    updatedAt: number;
+    completedAt?: number;
+    metadata?: Record<string, unknown>;
+}
+interface AnnotationItem {
+    id: string;
+    taskId: string;
+    data: Record<string, unknown>;
+    status: AnnotationItemStatus;
+    priority: number;
+    assignedTo?: string[];
+    annotations: Annotation[];
+    consensus?: ConsensusResult;
+    createdAt: number;
+    updatedAt: number;
+    metadata?: Record<string, unknown>;
+}
+interface Annotation {
+    id: string;
+    itemId: string;
+    annotatorId: string;
+    value: Record<string, unknown>;
+    explanation?: string;
+    confidence?: number;
+    duration: number;
+    createdAt: number;
+    isGold?: boolean;
+    quality?: AnnotationQuality;
+}
+interface AnnotationQuality {
+    score: number;
+    agreementWithConsensus?: number;
+    flagged?: boolean;
+    flagReason?: string;
+}
+interface Annotator {
+    id: string;
+    name: string;
+    email?: string;
+    role: 'annotator' | 'reviewer' | 'expert' | 'admin';
+    accuracy?: number;
+    completedCount: number;
+    activeTaskIds: string[];
+    metadata?: Record<string, unknown>;
+}
+interface AnnotatorStats {
+    annotatorId: string;
+    totalAnnotations: number;
+    averageDuration: number;
+    accuracy: number;
+    agreementRate: number;
+    flaggedRate: number;
+    byTask: Record<string, {
+        annotations: number;
+        accuracy: number;
+        avgDuration: number;
+    }>;
+}
+interface AnnotationQueueConfig {
+    task: IAnnotationTask;
+    items: AnnotationItem[];
+    prioritization?: PrioritizationType;
+    assignment?: AssignmentStrategy;
+    qualityControl?: QualityControlConfig;
+}
+type PrioritizationType = 'fifo' | 'uncertainty' | 'diversity' | 'importance' | 'custom';
+type AssignmentStrategy = 'round-robin' | 'expertise' | 'load-balanced' | 'random';
+interface QualityControlConfig {
+    goldStandard?: GoldStandardItem[];
+    goldRatio?: number;
+    agreementThreshold?: number;
+    minAnnotatorAccuracy?: number;
+    autoReject?: boolean;
+    expertReviewThreshold?: number;
+}
+interface GoldStandardItem {
+    itemId: string;
+    expectedAnnotation: Record<string, unknown>;
+    tolerance?: Record<string, number>;
+}
+type ConsensusMethod = 'majority' | 'unanimous' | 'weighted' | 'dawid-skene' | 'expert';
+interface ConsensusConfig {
+    method: ConsensusMethod;
+    minAgreement?: number;
+    weights?: Record<string, number>;
+    expertAnnotatorId?: string;
+    tieBreaker?: 'expert' | 'random' | 'none';
+}
+interface ConsensusResult {
+    value: Record<string, unknown>;
+    method: ConsensusMethod;
+    agreement: number;
+    confidence: number;
+    contributingAnnotations: string[];
+    disagreements?: Disagreement[];
+}
+interface Disagreement {
+    field: string;
+    values: Array<{
+        value: unknown;
+        annotatorIds: string[];
+        count: number;
+    }>;
+    resolved: boolean;
+    resolution?: unknown;
+}
+interface QueueStats {
+    taskId: string;
+    totalItems: number;
+    pendingItems: number;
+    assignedItems: number;
+    completedItems: number;
+    flaggedItems: number;
+    averageAnnotationsPerItem: number;
+    averageAgreement: number;
+    estimatedCompletion?: Date;
+}
+interface AnnotationResults {
+    taskId: string;
+    items: Array<{
+        itemId: string;
+        data: Record<string, unknown>;
+        consensus: ConsensusResult;
+        annotations: Annotation[];
+    }>;
+    stats: {
+        total: number;
+        completed: number;
+        agreementRate: number;
+        flaggedCount: number;
+        avgAnnotationsPerItem: number;
+    };
+}
+interface BatchAssignment {
+    annotatorId: string;
+    itemIds: string[];
+    deadline?: Date;
+    instructions?: string;
+}
+type AnnotationEventType = 'task:created' | 'task:started' | 'task:paused' | 'task:completed' | 'item:assigned' | 'item:annotated' | 'item:flagged' | 'item:consensus_reached' | 'annotator:joined' | 'annotator:warning' | 'quality:alert';
+interface AnnotationEvent {
+    type: AnnotationEventType;
+    taskId: string;
+    itemId?: string;
+    annotatorId?: string;
+    data?: Record<string, unknown>;
+    timestamp: number;
+}
+
+declare class AnnotationTask implements IAnnotationTask {
+    readonly id: string;
+    readonly name: string;
+    readonly description: string;
+    readonly instructions: string;
+    readonly schema: z.ZodSchema;
+    status: AnnotationTaskStatus;
+    readonly itemsPerAnnotator: number;
+    readonly annotatorsPerItem: number;
+    readonly deadline?: Date;
+    readonly createdAt: number;
+    updatedAt: number;
+    completedAt?: number;
+    readonly metadata?: Record<string, unknown>;
+    constructor(config: AnnotationTaskConfig);
+    start(): void;
+    pause(): void;
+    resume(): void;
+    complete(): void;
+    cancel(): void;
+    validateAnnotation(value: unknown): {
+        valid: boolean;
+        error?: string;
+    };
+    isPastDeadline(): boolean;
+    toConfig(): AnnotationTaskConfig;
+}
+declare function createAnnotationTask(config: AnnotationTaskConfig): AnnotationTask;
+declare const BinaryClassificationSchema: z.ZodObject<{
+    label: z.ZodEnum<["positive", "negative"]>;
+    confidence: z.ZodOptional<z.ZodNumber>;
+    notes: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    label: "positive" | "negative";
+    confidence?: number | undefined;
+    notes?: string | undefined;
+}, {
+    label: "positive" | "negative";
+    confidence?: number | undefined;
+    notes?: string | undefined;
+}>;
+declare const QualityRatingSchema: z.ZodObject<{
+    accuracy: z.ZodNumber;
+    helpfulness: z.ZodNumber;
+    safety: z.ZodEnum<["pass", "fail"]>;
+    corrections: z.ZodOptional<z.ZodString>;
+    notes: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    accuracy: number;
+    helpfulness: number;
+    safety: "pass" | "fail";
+    notes?: string | undefined;
+    corrections?: string | undefined;
+}, {
+    accuracy: number;
+    helpfulness: number;
+    safety: "pass" | "fail";
+    notes?: string | undefined;
+    corrections?: string | undefined;
+}>;
+declare const TextSpanSchema: z.ZodObject<{
+    spans: z.ZodArray<z.ZodObject<{
+        start: z.ZodNumber;
+        end: z.ZodNumber;
+        label: z.ZodString;
+        text: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        start: number;
+        label: string;
+        end: number;
+        text?: string | undefined;
+    }, {
+        start: number;
+        label: string;
+        end: number;
+        text?: string | undefined;
+    }>, "many">;
+    notes: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    spans: {
+        start: number;
+        label: string;
+        end: number;
+        text?: string | undefined;
+    }[];
+    notes?: string | undefined;
+}, {
+    spans: {
+        start: number;
+        label: string;
+        end: number;
+        text?: string | undefined;
+    }[];
+    notes?: string | undefined;
+}>;
+
+interface QueueEvents {
+    'item:assigned': (itemId: string, annotatorId: string) => void;
+    'item:annotated': (itemId: string, annotation: Annotation) => void;
+    'item:consensus': (itemId: string, consensus: ConsensusResult) => void;
+    'item:flagged': (itemId: string, reason: string) => void;
+}
+declare class AnnotationQueue extends EventEmitter<QueueEvents> {
+    readonly task: AnnotationTask;
+    private items;
+    private annotatorAssignments;
+    private annotatorCounts;
+    constructor(config: AnnotationQueueConfig);
+    getNextItem(annotatorId: string): AnnotationItem | null;
+    assignItem(itemId: string, annotatorId: string): void;
+    submitAnnotation(itemId: string, annotatorId: string, value: Record<string, unknown>, duration: number): Annotation;
+    flagItem(itemId: string, reason: string): void;
+    skipItem(itemId: string, annotatorId: string): void;
+    getBatchAssignment(annotatorId: string, count: number): BatchAssignment;
+    getStats(): QueueStats;
+    getItem(itemId: string): AnnotationItem | undefined;
+    getItems(): AnnotationItem[];
+    getItemsByStatus(status: AnnotationItemStatus): AnnotationItem[];
+    getAnnotatorCount(annotatorId: string): number;
+}
+declare function createAnnotationQueue(config: AnnotationQueueConfig): AnnotationQueue;
+
+declare class ConsensusManager {
+    private method;
+    private weights?;
+    private expertAnnotatorId?;
+    constructor(config: ConsensusConfig);
+    calculateConsensus(annotations: Annotation[]): ConsensusResult;
+    private majorityConsensus;
+    private findMajority;
+    private unanimousConsensus;
+    private weightedConsensus;
+    private expertConsensus;
+    calculateAgreement(annotations: Annotation[]): number;
+}
+declare function createConsensusManager(config: ConsensusConfig): ConsensusManager;
+
+export { type AnnotationTaskStatus as A, type BatchAssignment as B, type ConsensusMethod as C, type Disagreement as D, type GoldStandardItem as G, type IAnnotationTask as I, type PrioritizationType as P, type QualityControlConfig as Q, TextSpanSchema as T, type AnnotationItemStatus as a, type AnnotationTaskConfig as b, type AnnotationItem as c, type Annotation as d, type AnnotationQuality as e, type Annotator as f, type AnnotatorStats as g, type AnnotationQueueConfig as h, type AssignmentStrategy as i, type ConsensusConfig as j, type ConsensusResult as k, type QueueStats as l, type AnnotationResults as m, type AnnotationEventType as n, type AnnotationEvent as o, AnnotationTask as p, createAnnotationTask as q, BinaryClassificationSchema as r, QualityRatingSchema as s, AnnotationQueue as t, createAnnotationQueue as u, ConsensusManager as v, createConsensusManager as w };