@dewtech/dare-cli 3.11.0 → 3.12.0

package/skills/dare-llm-integration/types.ts

@@ -0,0 +1,91 @@
+/**
+ * dare-llm-integration — shared types
+ * License: MIT
+ */
+
+export interface CompletionRequest {
+  model: string;
+  prompt: string;
+  temperature?: number;
+  maxTokens?: number;
+  systemPrompt?: string;
+  timeoutMs?: number;
+}
+
+export interface CompletionResponse {
+  text: string;
+  tokensUsed: {
+    input: number;
+    output: number;
+    total: number;
+  };
+  cached: boolean;
+  model: string;
+  latencyMs?: number;
+}
+
+export interface EmbedRequest {
+  model: string;
+  input: string | string[];
+  timeoutMs?: number;
+}
+
+export interface EmbeddingResponse {
+  embeddings: number[][];
+  tokensUsed: number;
+  model: string;
+  cached: boolean;
+}
+
+export interface LLMProvider {
+  complete(request: CompletionRequest): Promise<CompletionResponse>;
+  embed(request: EmbedRequest): Promise<EmbeddingResponse>;
+}
+
+export interface PromptTemplate {
+  name: string;
+  version: string;
+  content: string;
+  variables: string[];
+}
+
+export interface CacheEntry {
+  value: CompletionResponse | EmbeddingResponse;
+  expiresAt: number;
+  createdAt: number;
+  hits: number;
+}
+
+export interface RateLimiterConfig {
+  rps: number;
+  maxBurst?: number;
+}
+
+export interface OutputValidationResult {
+  ok: boolean;
+  data?: unknown;
+  errors: OutputValidationError[];
+}
+
+export interface OutputValidationError {
+  field: string;
+  message: string;
+  value?: unknown;
+}
+
+export interface MetricResult {
+  id: string;
+  pass: boolean;
+  description: string;
+  detail?: string;
+}
+
+export interface LLMUsageStats {
+  totalRequests: number;
+  cacheHits: number;
+  cacheMisses: number;
+  rateLimitedRequests: number;
+  validatedResponses: number;
+  totalInputTokens: number;
+  totalOutputTokens: number;
+}

package/skills/dare-llm-integration/validators/output_validator.ts

@@ -0,0 +1,200 @@
+/**
+ * dare-llm-integration — OutputValidator
+ * Validates LLM output against a JSON Schema.
+ * Zero external dependencies — uses a minimal built-in schema checker.
+ * License: MIT
+ */
+
+import type { OutputValidationResult, OutputValidationError } from '../types.js';
+
+type JsonSchemaType = 'string' | 'number' | 'boolean' | 'object' | 'array' | 'null' | 'integer';
+
+export interface JsonSchema {
+  type?: JsonSchemaType | JsonSchemaType[];
+  properties?: Record<string, JsonSchema>;
+  required?: string[];
+  items?: JsonSchema;
+  minLength?: number;
+  maxLength?: number;
+  minimum?: number;
+  maximum?: number;
+  enum?: unknown[];
+  format?: string;
+  additionalProperties?: boolean | JsonSchema;
+  [key: string]: unknown;
+}
+
+export class OutputValidator {
+  /**
+   * Parse the output string as JSON, then validate against schema.
+   *
+   * @param output - Raw string output from LLM
+   * @param schema - JSON Schema object to validate against
+   */
+  validate(output: string, schema: JsonSchema): OutputValidationResult {
+    // Step 1: Parse JSON
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(output);
+    } catch {
+      return {
+        ok: false,
+        errors: [
+          {
+            field: '$root',
+            message: `Output is not valid JSON: ${output.slice(0, 100)}`,
+          },
+        ],
+      };
+    }
+
+    // Step 2: Validate against schema
+    const errors: OutputValidationError[] = [];
+    validateValue(parsed, schema, '$', errors);
+
+    if (errors.length > 0) {
+      return { ok: false, errors };
+    }
+
+    return { ok: true, data: parsed, errors: [] };
+  }
+
+  /**
+   * Validate a pre-parsed value (skip JSON.parse step).
+   */
+  validateParsed(value: unknown, schema: JsonSchema): OutputValidationResult {
+    const errors: OutputValidationError[] = [];
+    validateValue(value, schema, '$', errors);
+
+    if (errors.length > 0) {
+      return { ok: false, errors };
+    }
+
+    return { ok: true, data: value, errors: [] };
+  }
+}
+
+function validateValue(
+  value: unknown,
+  schema: JsonSchema,
+  path: string,
+  errors: OutputValidationError[]
+): void {
+  // Type check
+  if (schema.type !== undefined) {
+    const types = Array.isArray(schema.type) ? schema.type : [schema.type];
+    const actualType = getJsonType(value);
+    const matches = types.some((t) => {
+      if (t === 'integer') return Number.isInteger(value);
+      return t === actualType;
+    });
+    if (!matches) {
+      errors.push({
+        field: path,
+        message: `Expected type ${types.join('|')}, got ${actualType}`,
+        value,
+      });
+      return; // No point checking further if wrong type
+    }
+  }
+
+  // Object-specific checks
+  if (schema.type === 'object' || (schema.properties !== undefined && typeof value === 'object' && value !== null && !Array.isArray(value))) {
+    const obj = value as Record<string, unknown>;
+
+    // Required fields
+    if (schema.required) {
+      for (const field of schema.required) {
+        if (!(field in obj) || obj[field] === undefined || obj[field] === null) {
+          errors.push({
+            field: `${path}.${field}`,
+            message: `Required field "${field}" is missing`,
+          });
+        }
+      }
+    }
+
+    // Properties
+    if (schema.properties) {
+      for (const [key, propSchema] of Object.entries(schema.properties)) {
+        if (key in obj) {
+          validateValue(obj[key], propSchema, `${path}.${key}`, errors);
+        }
+      }
+    }
+  }
+
+  // Array-specific checks
+  if (schema.type === 'array' || (schema.items !== undefined && Array.isArray(value))) {
+    const arr = value as unknown[];
+    if (schema.items) {
+      arr.forEach((item, idx) => {
+        validateValue(item, schema.items!, `${path}[${idx}]`, errors);
+      });
+    }
+  }
+
+  // String constraints
+  if (typeof value === 'string') {
+    if (schema.minLength !== undefined && value.length < schema.minLength) {
+      errors.push({
+        field: path,
+        message: `String length ${value.length} is less than minLength ${schema.minLength}`,
+        value,
+      });
+    }
+    if (schema.maxLength !== undefined && value.length > schema.maxLength) {
+      errors.push({
+        field: path,
+        message: `String length ${value.length} exceeds maxLength ${schema.maxLength}`,
+        value,
+      });
+    }
+  }
+
+  // Number constraints
+  if (typeof value === 'number') {
+    if (schema.minimum !== undefined && value < schema.minimum) {
+      errors.push({
+        field: path,
+        message: `Value ${value} is less than minimum ${schema.minimum}`,
+        value,
+      });
+    }
+    if (schema.maximum !== undefined && value > schema.maximum) {
+      errors.push({
+        field: path,
+        message: `Value ${value} exceeds maximum ${schema.maximum}`,
+        value,
+      });
+    }
+  }
+
+  // Enum check
+  if (schema.enum !== undefined) {
+    const enumValues = schema.enum;
+    const found = enumValues.some((e) => deepEqual(e, value));
+    if (!found) {
+      errors.push({
+        field: path,
+        message: `Value not in enum: ${JSON.stringify(enumValues)}`,
+        value,
+      });
+    }
+  }
+}
+
+function getJsonType(value: unknown): JsonSchemaType {
+  if (value === null) return 'null';
+  if (Array.isArray(value)) return 'array';
+  return typeof value as JsonSchemaType;
+}
+
+function deepEqual(a: unknown, b: unknown): boolean {
+  if (a === b) return true;
+  if (typeof a !== typeof b) return false;
+  if (typeof a === 'object' && a !== null && b !== null) {
+    return JSON.stringify(a) === JSON.stringify(b);
+  }
+  return false;
+}

package/skills/dare-quality-telemetry/collect.ts

@@ -0,0 +1,134 @@
+/**
+ * dare-quality-telemetry — main collect function
+ * Orchestrates metric collection for all configured skills and saves results.
+ * License: MIT
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { execSync } from 'child_process';
+import {
+  CollectorConfig,
+  ProjectMetricReport,
+  SkillMetricReport,
+  MetricResult,
+} from './types.js';
+import { collectors } from './collectors/index.js';
+
+/**
+ * Detects the current git commit SHA.
+ * Returns "unknown" if git is not available or the directory is not a repo.
+ */
+export function detectCommit(projectPath: string): string {
+  try {
+    const result = execSync('git rev-parse --short HEAD', {
+      cwd: projectPath,
+      encoding: 'utf-8',
+      stdio: ['pipe', 'pipe', 'pipe'],
+    });
+    return result.trim();
+  } catch {
+    return 'unknown';
+  }
+}
+
+/**
+ * Builds a summary object from a list of MetricResults.
+ */
+export function buildSummary(metrics: MetricResult[]): {
+  passed: number;
+  total: number;
+  score: string;
+  allPass: boolean;
+} {
+  const total = metrics.length;
+  const passed = metrics.filter((m) => m.pass).length;
+  const score = total === 0 ? '0/0' : `${passed}/${total}`;
+  const allPass = total > 0 && passed === total;
+  return { passed, total, score, allPass };
+}
+
+/**
+ * Main function: collects metrics for all configured skills in the project.
+ *
+ * @param config - CollectorConfig specifying project path and skills to validate.
+ * @returns ProjectMetricReport with all results.
+ *
+ * Side effect: writes result to <projectPath>/tmp/dare_metrics.json
+ */
+export async function collectMetrics(config: CollectorConfig): Promise<ProjectMetricReport> {
+  const { projectPath, skills } = config;
+  const timestamp = new Date().toISOString();
+  const commit = detectCommit(projectPath);
+
+  const skillReports: SkillMetricReport[] = [];
+
+  for (const skillName of skills) {
+    const collector = collectors[skillName];
+
+    if (!collector) {
+      // Unknown skill — report it as a failed collector with a placeholder metric
+      skillReports.push({
+        skillName,
+        timestamp,
+        commit,
+        metrics: [
+          {
+            id: 'M-00',
+            pass: false,
+            description: 'Collector available',
+            details: `No collector registered for skill "${skillName}". Available: ${Object.keys(collectors).join(', ')}`,
+          },
+        ],
+        summary: { passed: 0, total: 1, score: '0/1', allPass: false },
+      });
+      continue;
+    }
+
+    let metrics: MetricResult[];
+    try {
+      metrics = await collector(projectPath);
+    } catch (err) {
+      metrics = [
+        {
+          id: 'M-00',
+          pass: false,
+          description: 'Collector executed without error',
+          details: `Collector for "${skillName}" threw: ${(err as Error).message}`,
+        },
+      ];
+    }
+
+    const summary = buildSummary(metrics);
+    skillReports.push({ skillName, timestamp, commit, metrics, summary });
+  }
+
+  // Calculate overall summary
+  const allMetrics = skillReports.flatMap((s) => s.metrics);
+  const overall = buildSummary(allMetrics);
+
+  const report: ProjectMetricReport = {
+    timestamp,
+    commit,
+    projectPath,
+    skills: skillReports,
+    overall,
+  };
+
+  // Save to tmp/dare_metrics.json
+  const tmpDir = path.join(projectPath, 'tmp');
+  try {
+    if (!fs.existsSync(tmpDir)) {
+      fs.mkdirSync(tmpDir, { recursive: true });
+    }
+    fs.writeFileSync(
+      path.join(tmpDir, 'dare_metrics.json'),
+      JSON.stringify(report, null, 2),
+      'utf-8'
+    );
+  } catch {
+    // Non-fatal: log failure but return the report regardless
+  }
+
+  return report;
+}