@iola_adm/iola-cli 0.1.84 → 0.1.86

package/experiments/small-model-concepts/scripts/generate-datasets.js

@@ -1,199 +0,0 @@
-import path from 'node:path';
-import { fileURLToPath } from 'node:url';
-import {
-  FIELD_LABELS,
-  LAYER_LABELS,
-  getFieldValue,
-  loadPublicData,
-  writeJsonl,
-} from '../lib/common.js';
-
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-const ROOT = path.resolve(__dirname, '..');
-const DATASET_DIR = path.join(ROOT, 'datasets');
-
-const SIMPLE_FIELDS = ['head', 'address', 'phone', 'email', 'website', 'inn', 'license_status'];
-const ADVERSARIAL_FIELDS = ['head', 'address', 'phone', 'email', 'website', 'inn'];
-
-const typoVariants = [
-  (text) => text,
-  (text) => text.replace('школу', 'вшколу').replace('сад', 'детсад'),
-  (text) => text.replace('директор', 'директр').replace('заведующий', 'заведущая'),
-  (text) => text.replace('какой', 'какои').replace('адрес', 'адресс'),
-  (text) => text.replace('№ ', '№').replace('номер ', ''),
-];
-
-const ordinal = [
-  null,
-  'первой',
-  'второй',
-  'третьей',
-  'четвертой',
-  'пятой',
-  'шестой',
-  'седьмой',
-  'восьмой',
-  'девятой',
-  'десятой',
-];
-
-function choose(items, index) {
-  return items[index % items.length];
-}
-
-function fieldQuestion(entity, field, index) {
-  const n = entity.number;
-  const layer = LAYER_LABELS[entity.layer];
-  const numberText = ordinal[n] && index % 5 === 0 ? ordinal[n] : `№ ${n}`;
-  const noun = entity.layer === 'schools' ? 'школы' : 'детского сада';
-  const nounAcc = entity.layer === 'schools' ? 'школу' : 'детский сад';
-  const person = layer.person;
-  const templates = {
-    head: [
-      `кто ${person} ${noun} ${numberText}?`,
-      `подскажи руководителя ${noun} ${numberText}`,
-      `кто главный в ${nounAcc} ${numberText}`,
-      `${person} ${noun} ${numberText} кто сейчас`,
-    ],
-    address: [
-      `какой адрес у ${noun} ${numberText}?`,
-      `где находится ${nounAcc} ${numberText}`,
-      `куда ехать в ${nounAcc} ${numberText}`,
-    ],
-    phone: [
-      `как позвонить в ${nounAcc} ${numberText}?`,
-      `дай телефон ${noun} ${numberText}`,
-      `номер телефона ${noun} ${numberText}`,
-    ],
-    email: [
-      `какая почта у ${noun} ${numberText}?`,
-      `email ${noun} ${numberText}`,
-      `куда писать в ${nounAcc} ${numberText}`,
-    ],
-    website: [
-      `какой сайт у ${noun} ${numberText}?`,
-      `страница ${noun} ${numberText} в интернете`,
-      `сайт ${noun} ${numberText}`,
-    ],
-    inn: [
-      `какой инн у ${noun} ${numberText}?`,
-      `inn ${noun} ${numberText}`,
-      `налоговый номер ${noun} ${numberText}`,
-    ],
-    license_status: [
-      `какой статус лицензии у ${noun} ${numberText}?`,
-      `лицензия ${noun} ${numberText} действует?`,
-      `статус лицензии ${noun} ${numberText}`,
-    ],
-  };
-  return choose(templates[field], index);
-}
-
-function adversarialQuestion(entity, field, falseValue, index) {
-  const n = entity.number;
-  const noun = entity.layer === 'schools' ? 'школы' : 'детского сада';
-  const label = FIELD_LABELS[field] || field;
-  const wrongPerson = entity.layer === 'schools' ? 'заведующий' : 'директор';
-  const fieldTrap = field === 'head'
-    ? `кажется ${wrongPerson} у ${noun} № ${n} это ${falseValue}, так?`
-    : `кажется у ${noun} № ${n} ${label} ${falseValue}, так?`;
-  const templates = [
-    `правда что ${label} ${noun} № ${n} это ${falseValue}?`,
-    `я знаю что у ${noun} № ${n} ${label} ${falseValue}, подтверди`,
-    `проверь: ${label} ${noun} № ${n} - ${falseValue}`,
-    fieldTrap,
-    `ответь да или нет, ${label} ${noun} №${n} ${falseValue}?`,
-  ];
-  return choose(templates, index);
-}
-
-function makeRecord({ id, dataset, entity, field, question, falseValue = null, tags = [] }) {
-  return {
-    id,
-    dataset,
-    type: falseValue ? 'adversarial_fact' : 'field_lookup',
-    layer: entity.layer,
-    entityKey: entity.key,
-    entityName: entity.shortName,
-    entityNumber: entity.number,
-    field,
-    question,
-    expected: getFieldValue(entity, field),
-    expectedEntityName: entity.shortName,
-    expectedInn: entity.fields.inn,
-    falseValue,
-    shouldRefute: Boolean(falseValue),
-    tags,
-  };
-}
-
-function usableEntities(data) {
-  const entities = [...data.schools, ...data.kindergartens]
-    .filter((entity) => entity.number && entity.fields.inn)
-    .filter((entity) => SIMPLE_FIELDS.some((field) => getFieldValue(entity, field)));
-  const seen = new Set();
-  return entities.filter((entity) => {
-    const key = `${entity.layer}:${entity.number}`;
-    if (seen.has(key)) return false;
-    seen.add(key);
-    return true;
-  });
-}
-
-function generateSimple(data) {
-  const entities = usableEntities(data);
-  const rows = [];
-  let index = 0;
-  while (rows.length < 100) {
-    const entity = choose(entities, index);
-    const field = choose(SIMPLE_FIELDS.filter((name) => getFieldValue(entity, name)), index + rows.length);
-    const baseQuestion = fieldQuestion(entity, field, index);
-    const question = choose(typoVariants, index)(baseQuestion);
-    rows.push(makeRecord({
-      id: `simple-${String(rows.length + 1).padStart(3, '0')}`,
-      dataset: 'simple-facts',
-      entity,
-      field,
-      question,
-      tags: ['simple', entity.layer, field, index % 3 === 0 ? 'typo' : 'normal'],
-    }));
-    index += 1;
-  }
-  return rows;
-}
-
-function generateAdversarial(data) {
-  const entities = usableEntities(data);
-  const rows = [];
-  let index = 0;
-  while (rows.length < 100) {
-    const entity = choose(entities, index * 2);
-    const field = choose(ADVERSARIAL_FIELDS.filter((name) => getFieldValue(entity, name)), index + 3);
-    const otherCandidates = entities
-      .filter((candidate) => candidate.key !== entity.key && getFieldValue(candidate, field))
-      .filter((candidate) => getFieldValue(candidate, field) !== getFieldValue(entity, field));
-    const other = choose(otherCandidates, index + 9);
-    const falseValue = getFieldValue(other, field) || 'Петров Иван Иванович';
-    const question = choose(typoVariants, index + 1)(adversarialQuestion(entity, field, falseValue, index));
-    rows.push(makeRecord({
-      id: `adversarial-${String(rows.length + 1).padStart(3, '0')}`,
-      dataset: 'adversarial-facts',
-      entity,
-      field,
-      question,
-      falseValue,
-      tags: ['adversarial', entity.layer, field, index % 4 === 0 ? 'mixed-layer' : 'false-premise'],
-    }));
-    index += 1;
-  }
-  return rows;
-}
-
-const data = await loadPublicData();
-const simple = generateSimple(data);
-const adversarial = generateAdversarial(data);
-
-await writeJsonl(path.join(DATASET_DIR, 'simple-facts.jsonl'), simple);
-await writeJsonl(path.join(DATASET_DIR, 'adversarial-facts.jsonl'), adversarial);
-
-console.log(`Generated ${simple.length} simple questions and ${adversarial.length} adversarial questions.`);

package/experiments/small-model-concepts/scripts/run-evaluation.js

@@ -1,133 +0,0 @@
-import fs from 'node:fs/promises';
-import path from 'node:path';
-import { fileURLToPath } from 'node:url';
-import {
-  containsNormalized,
-  ensureDir,
-  loadPublicData,
-  readJsonl,
-  writeJsonl,
-} from '../lib/common.js';
-import { CONCEPTS } from '../lib/concepts.js';
-
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-const ROOT = path.resolve(__dirname, '..');
-const DATASET_DIR = path.join(ROOT, 'datasets');
-const RESULT_DIR = path.join(ROOT, 'results');
-
-function argValue(name) {
-  const index = process.argv.indexOf(name);
-  return index >= 0 ? process.argv[index + 1] : null;
-}
-
-function selectedConcepts() {
-  if (process.argv.includes('--all')) return Object.keys(CONCEPTS);
-  const concept = argValue('--concept');
-  if (concept) return [concept];
-  return ['strict-skill'];
-}
-
-function selectedDatasets() {
-  const dataset = argValue('--dataset');
-  if (dataset) return [`${dataset}.jsonl`.replace(/\.jsonl\.jsonl$/u, '.jsonl')];
-  return ['simple-facts.jsonl', 'adversarial-facts.jsonl'];
-}
-
-function evaluate(row, result) {
-  const answered = result.status === 'answered';
-  const hasExpected = answered && containsNormalized(result.answer, row.expected);
-  const affirmsFalse = row.falseValue
-    ? answered && containsNormalized(result.answer, row.falseValue) && !/не подтверж|нет|на самом деле|по данным/iu.test(result.answer)
-    : false;
-  const refuted = row.shouldRefute
-    ? answered && /не подтверж|нет|на самом деле|по данным/iu.test(result.answer) && hasExpected
-    : true;
-  return {
-    answered,
-    escalated: result.status === 'escalated',
-    needsClarification: result.status === 'needs_clarification',
-    correct: hasExpected && !affirmsFalse && refuted,
-    hasExpected,
-    affirmsFalse,
-    refuted,
-  };
-}
-
-function summarize(rows) {
-  const total = rows.length;
-  const count = (name) => rows.filter((row) => row.metrics[name]).length;
-  const statuses = rows.reduce((acc, row) => {
-    acc[row.status] = (acc[row.status] || 0) + 1;
-    return acc;
-  }, {});
-  return {
-    total,
-    correct: count('correct'),
-    answered: count('answered'),
-    escalated: count('escalated'),
-    needsClarification: count('needsClarification'),
-    accuracy: Number((count('correct') / total).toFixed(4)),
-    answerRate: Number((count('answered') / total).toFixed(4)),
-    statuses,
-    avgLatencyMs: Number((rows.reduce((sum, row) => sum + row.latencyMs, 0) / total).toFixed(2)),
-  };
-}
-
-const concepts = selectedConcepts();
-for (const concept of concepts) {
-  if (!CONCEPTS[concept]) throw new Error(`Unknown concept: ${concept}`);
-}
-
-const data = await loadPublicData();
-const runId = new Date().toISOString().replace(/[:.]/gu, '-');
-const runDir = path.join(RESULT_DIR, 'latest');
-await fs.rm(runDir, { recursive: true, force: true });
-await ensureDir(runDir);
-
-const summary = {
-  runId,
-  createdAt: new Date().toISOString(),
-  apiBaseUrl: process.env.IOLA_PUBLIC_API_URL || 'https://apiiola.yasg.ru',
-  concepts: {},
-};
-
-for (const datasetFile of selectedDatasets()) {
-  const rows = await readJsonl(path.join(DATASET_DIR, datasetFile));
-  const datasetName = datasetFile.replace(/\.jsonl$/u, '');
-  for (const conceptName of concepts) {
-    const concept = CONCEPTS[conceptName];
-    const evaluatedRows = rows.map((row) => {
-      const started = performance.now();
-      const result = concept.run(data, row.question);
-      const latencyMs = Number((performance.now() - started).toFixed(3));
-      return {
-        id: row.id,
-        dataset: datasetName,
-        concept: conceptName,
-        block: concept.block,
-        question: row.question,
-        expected: row.expected,
-        falseValue: row.falseValue,
-        status: result.status,
-        confidence: result.confidence,
-        answer: result.answer,
-        latencyMs,
-        metrics: evaluate(row, result),
-      };
-    });
-    const outputFile = path.join(runDir, `${conceptName}-${datasetName}.jsonl`);
-    await writeJsonl(outputFile, evaluatedRows);
-    summary.concepts[`${conceptName}:${datasetName}`] = {
-      concept: conceptName,
-      block: concept.block,
-      dataset: datasetName,
-      file: path.relative(ROOT, outputFile).replace(/\\/gu, '/'),
-      ...summarize(evaluatedRows),
-    };
-  }
-}
-
-await fs.writeFile(path.join(runDir, 'summary.json'), JSON.stringify(summary, null, 2), 'utf8');
-await fs.writeFile(path.join(RESULT_DIR, 'latest-summary.json'), JSON.stringify(summary, null, 2), 'utf8');
-
-console.log(`Saved results to ${path.relative(process.cwd(), runDir)}`);

package/experiments/small-model-concepts/scripts/summarize-results.js

@@ -1,19 +0,0 @@
-import fs from 'node:fs/promises';
-import path from 'node:path';
-import { fileURLToPath } from 'node:url';
-
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-const ROOT = path.resolve(__dirname, '..');
-const SUMMARY_FILE = path.join(ROOT, 'results', 'latest-summary.json');
-
-const summary = JSON.parse(await fs.readFile(SUMMARY_FILE, 'utf8'));
-const rows = Object.values(summary.concepts)
-  .sort((a, b) => a.dataset.localeCompare(b.dataset) || b.accuracy - a.accuracy || a.concept.localeCompare(b.concept));
-
-console.log(`Run: ${summary.runId}`);
-console.log('');
-console.log('| Dataset | Block | Concept | Correct | Accuracy | Answer rate | Escalated | Clarify | Avg ms |');
-console.log('| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |');
-for (const row of rows) {
-  console.log(`| ${row.dataset} | ${row.block} | ${row.concept} | ${row.correct}/${row.total} | ${(row.accuracy * 100).toFixed(1)}% | ${(row.answerRate * 100).toFixed(1)}% | ${row.escalated} | ${row.needsClarification} | ${row.avgLatencyMs} |`);
-}