@iola_adm/iola-cli 0.1.82 → 0.1.84

package/experiments/small-model-concepts/scripts/generate-datasets.js

@@ -0,0 +1,199 @@
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import {
+  FIELD_LABELS,
+  LAYER_LABELS,
+  getFieldValue,
+  loadPublicData,
+  writeJsonl,
+} from '../lib/common.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const ROOT = path.resolve(__dirname, '..');
+const DATASET_DIR = path.join(ROOT, 'datasets');
+
+const SIMPLE_FIELDS = ['head', 'address', 'phone', 'email', 'website', 'inn', 'license_status'];
+const ADVERSARIAL_FIELDS = ['head', 'address', 'phone', 'email', 'website', 'inn'];
+
+const typoVariants = [
+  (text) => text,
+  (text) => text.replace('школу', 'вшколу').replace('сад', 'детсад'),
+  (text) => text.replace('директор', 'директр').replace('заведующий', 'заведущая'),
+  (text) => text.replace('какой', 'какои').replace('адрес', 'адресс'),
+  (text) => text.replace('№ ', '№').replace('номер ', ''),
+];
+
+const ordinal = [
+  null,
+  'первой',
+  'второй',
+  'третьей',
+  'четвертой',
+  'пятой',
+  'шестой',
+  'седьмой',
+  'восьмой',
+  'девятой',
+  'десятой',
+];
+
+function choose(items, index) {
+  return items[index % items.length];
+}
+
+function fieldQuestion(entity, field, index) {
+  const n = entity.number;
+  const layer = LAYER_LABELS[entity.layer];
+  const numberText = ordinal[n] && index % 5 === 0 ? ordinal[n] : `№ ${n}`;
+  const noun = entity.layer === 'schools' ? 'школы' : 'детского сада';
+  const nounAcc = entity.layer === 'schools' ? 'школу' : 'детский сад';
+  const person = layer.person;
+  const templates = {
+    head: [
+      `кто ${person} ${noun} ${numberText}?`,
+      `подскажи руководителя ${noun} ${numberText}`,
+      `кто главный в ${nounAcc} ${numberText}`,
+      `${person} ${noun} ${numberText} кто сейчас`,
+    ],
+    address: [
+      `какой адрес у ${noun} ${numberText}?`,
+      `где находится ${nounAcc} ${numberText}`,
+      `куда ехать в ${nounAcc} ${numberText}`,
+    ],
+    phone: [
+      `как позвонить в ${nounAcc} ${numberText}?`,
+      `дай телефон ${noun} ${numberText}`,
+      `номер телефона ${noun} ${numberText}`,
+    ],
+    email: [
+      `какая почта у ${noun} ${numberText}?`,
+      `email ${noun} ${numberText}`,
+      `куда писать в ${nounAcc} ${numberText}`,
+    ],
+    website: [
+      `какой сайт у ${noun} ${numberText}?`,
+      `страница ${noun} ${numberText} в интернете`,
+      `сайт ${noun} ${numberText}`,
+    ],
+    inn: [
+      `какой инн у ${noun} ${numberText}?`,
+      `inn ${noun} ${numberText}`,
+      `налоговый номер ${noun} ${numberText}`,
+    ],
+    license_status: [
+      `какой статус лицензии у ${noun} ${numberText}?`,
+      `лицензия ${noun} ${numberText} действует?`,
+      `статус лицензии ${noun} ${numberText}`,
+    ],
+  };
+  return choose(templates[field], index);
+}
+
+function adversarialQuestion(entity, field, falseValue, index) {
+  const n = entity.number;
+  const noun = entity.layer === 'schools' ? 'школы' : 'детского сада';
+  const label = FIELD_LABELS[field] || field;
+  const wrongPerson = entity.layer === 'schools' ? 'заведующий' : 'директор';
+  const fieldTrap = field === 'head'
+    ? `кажется ${wrongPerson} у ${noun} № ${n} это ${falseValue}, так?`
+    : `кажется у ${noun} № ${n} ${label} ${falseValue}, так?`;
+  const templates = [
+    `правда что ${label} ${noun} № ${n} это ${falseValue}?`,
+    `я знаю что у ${noun} № ${n} ${label} ${falseValue}, подтверди`,
+    `проверь: ${label} ${noun} № ${n} - ${falseValue}`,
+    fieldTrap,
+    `ответь да или нет, ${label} ${noun} №${n} ${falseValue}?`,
+  ];
+  return choose(templates, index);
+}
+
+function makeRecord({ id, dataset, entity, field, question, falseValue = null, tags = [] }) {
+  return {
+    id,
+    dataset,
+    type: falseValue ? 'adversarial_fact' : 'field_lookup',
+    layer: entity.layer,
+    entityKey: entity.key,
+    entityName: entity.shortName,
+    entityNumber: entity.number,
+    field,
+    question,
+    expected: getFieldValue(entity, field),
+    expectedEntityName: entity.shortName,
+    expectedInn: entity.fields.inn,
+    falseValue,
+    shouldRefute: Boolean(falseValue),
+    tags,
+  };
+}
+
+function usableEntities(data) {
+  const entities = [...data.schools, ...data.kindergartens]
+    .filter((entity) => entity.number && entity.fields.inn)
+    .filter((entity) => SIMPLE_FIELDS.some((field) => getFieldValue(entity, field)));
+  const seen = new Set();
+  return entities.filter((entity) => {
+    const key = `${entity.layer}:${entity.number}`;
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+}
+
+function generateSimple(data) {
+  const entities = usableEntities(data);
+  const rows = [];
+  let index = 0;
+  while (rows.length < 100) {
+    const entity = choose(entities, index);
+    const field = choose(SIMPLE_FIELDS.filter((name) => getFieldValue(entity, name)), index + rows.length);
+    const baseQuestion = fieldQuestion(entity, field, index);
+    const question = choose(typoVariants, index)(baseQuestion);
+    rows.push(makeRecord({
+      id: `simple-${String(rows.length + 1).padStart(3, '0')}`,
+      dataset: 'simple-facts',
+      entity,
+      field,
+      question,
+      tags: ['simple', entity.layer, field, index % 3 === 0 ? 'typo' : 'normal'],
+    }));
+    index += 1;
+  }
+  return rows;
+}
+
+function generateAdversarial(data) {
+  const entities = usableEntities(data);
+  const rows = [];
+  let index = 0;
+  while (rows.length < 100) {
+    const entity = choose(entities, index * 2);
+    const field = choose(ADVERSARIAL_FIELDS.filter((name) => getFieldValue(entity, name)), index + 3);
+    const otherCandidates = entities
+      .filter((candidate) => candidate.key !== entity.key && getFieldValue(candidate, field))
+      .filter((candidate) => getFieldValue(candidate, field) !== getFieldValue(entity, field));
+    const other = choose(otherCandidates, index + 9);
+    const falseValue = getFieldValue(other, field) || 'Петров Иван Иванович';
+    const question = choose(typoVariants, index + 1)(adversarialQuestion(entity, field, falseValue, index));
+    rows.push(makeRecord({
+      id: `adversarial-${String(rows.length + 1).padStart(3, '0')}`,
+      dataset: 'adversarial-facts',
+      entity,
+      field,
+      question,
+      falseValue,
+      tags: ['adversarial', entity.layer, field, index % 4 === 0 ? 'mixed-layer' : 'false-premise'],
+    }));
+    index += 1;
+  }
+  return rows;
+}
+
+const data = await loadPublicData();
+const simple = generateSimple(data);
+const adversarial = generateAdversarial(data);
+
+await writeJsonl(path.join(DATASET_DIR, 'simple-facts.jsonl'), simple);
+await writeJsonl(path.join(DATASET_DIR, 'adversarial-facts.jsonl'), adversarial);
+
+console.log(`Generated ${simple.length} simple questions and ${adversarial.length} adversarial questions.`);

package/experiments/small-model-concepts/scripts/run-evaluation.js

@@ -0,0 +1,133 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import {
+  containsNormalized,
+  ensureDir,
+  loadPublicData,
+  readJsonl,
+  writeJsonl,
+} from '../lib/common.js';
+import { CONCEPTS } from '../lib/concepts.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const ROOT = path.resolve(__dirname, '..');
+const DATASET_DIR = path.join(ROOT, 'datasets');
+const RESULT_DIR = path.join(ROOT, 'results');
+
+function argValue(name) {
+  const index = process.argv.indexOf(name);
+  return index >= 0 ? process.argv[index + 1] : null;
+}
+
+function selectedConcepts() {
+  if (process.argv.includes('--all')) return Object.keys(CONCEPTS);
+  const concept = argValue('--concept');
+  if (concept) return [concept];
+  return ['strict-skill'];
+}
+
+function selectedDatasets() {
+  const dataset = argValue('--dataset');
+  if (dataset) return [`${dataset}.jsonl`.replace(/\.jsonl\.jsonl$/u, '.jsonl')];
+  return ['simple-facts.jsonl', 'adversarial-facts.jsonl'];
+}
+
+function evaluate(row, result) {
+  const answered = result.status === 'answered';
+  const hasExpected = answered && containsNormalized(result.answer, row.expected);
+  const affirmsFalse = row.falseValue
+    ? answered && containsNormalized(result.answer, row.falseValue) && !/не подтверж|нет|на самом деле|по данным/iu.test(result.answer)
+    : false;
+  const refuted = row.shouldRefute
+    ? answered && /не подтверж|нет|на самом деле|по данным/iu.test(result.answer) && hasExpected
+    : true;
+  return {
+    answered,
+    escalated: result.status === 'escalated',
+    needsClarification: result.status === 'needs_clarification',
+    correct: hasExpected && !affirmsFalse && refuted,
+    hasExpected,
+    affirmsFalse,
+    refuted,
+  };
+}
+
+function summarize(rows) {
+  const total = rows.length;
+  const count = (name) => rows.filter((row) => row.metrics[name]).length;
+  const statuses = rows.reduce((acc, row) => {
+    acc[row.status] = (acc[row.status] || 0) + 1;
+    return acc;
+  }, {});
+  return {
+    total,
+    correct: count('correct'),
+    answered: count('answered'),
+    escalated: count('escalated'),
+    needsClarification: count('needsClarification'),
+    accuracy: Number((count('correct') / total).toFixed(4)),
+    answerRate: Number((count('answered') / total).toFixed(4)),
+    statuses,
+    avgLatencyMs: Number((rows.reduce((sum, row) => sum + row.latencyMs, 0) / total).toFixed(2)),
+  };
+}
+
+const concepts = selectedConcepts();
+for (const concept of concepts) {
+  if (!CONCEPTS[concept]) throw new Error(`Unknown concept: ${concept}`);
+}
+
+const data = await loadPublicData();
+const runId = new Date().toISOString().replace(/[:.]/gu, '-');
+const runDir = path.join(RESULT_DIR, 'latest');
+await fs.rm(runDir, { recursive: true, force: true });
+await ensureDir(runDir);
+
+const summary = {
+  runId,
+  createdAt: new Date().toISOString(),
+  apiBaseUrl: process.env.IOLA_PUBLIC_API_URL || 'https://apiiola.yasg.ru',
+  concepts: {},
+};
+
+for (const datasetFile of selectedDatasets()) {
+  const rows = await readJsonl(path.join(DATASET_DIR, datasetFile));
+  const datasetName = datasetFile.replace(/\.jsonl$/u, '');
+  for (const conceptName of concepts) {
+    const concept = CONCEPTS[conceptName];
+    const evaluatedRows = rows.map((row) => {
+      const started = performance.now();
+      const result = concept.run(data, row.question);
+      const latencyMs = Number((performance.now() - started).toFixed(3));
+      return {
+        id: row.id,
+        dataset: datasetName,
+        concept: conceptName,
+        block: concept.block,
+        question: row.question,
+        expected: row.expected,
+        falseValue: row.falseValue,
+        status: result.status,
+        confidence: result.confidence,
+        answer: result.answer,
+        latencyMs,
+        metrics: evaluate(row, result),
+      };
+    });
+    const outputFile = path.join(runDir, `${conceptName}-${datasetName}.jsonl`);
+    await writeJsonl(outputFile, evaluatedRows);
+    summary.concepts[`${conceptName}:${datasetName}`] = {
+      concept: conceptName,
+      block: concept.block,
+      dataset: datasetName,
+      file: path.relative(ROOT, outputFile).replace(/\\/gu, '/'),
+      ...summarize(evaluatedRows),
+    };
+  }
+}
+
+await fs.writeFile(path.join(runDir, 'summary.json'), JSON.stringify(summary, null, 2), 'utf8');
+await fs.writeFile(path.join(RESULT_DIR, 'latest-summary.json'), JSON.stringify(summary, null, 2), 'utf8');
+
+console.log(`Saved results to ${path.relative(process.cwd(), runDir)}`);

package/experiments/small-model-concepts/scripts/summarize-results.js

@@ -0,0 +1,19 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const ROOT = path.resolve(__dirname, '..');
+const SUMMARY_FILE = path.join(ROOT, 'results', 'latest-summary.json');
+
+const summary = JSON.parse(await fs.readFile(SUMMARY_FILE, 'utf8'));
+const rows = Object.values(summary.concepts)
+  .sort((a, b) => a.dataset.localeCompare(b.dataset) || b.accuracy - a.accuracy || a.concept.localeCompare(b.concept));
+
+console.log(`Run: ${summary.runId}`);
+console.log('');
+console.log('| Dataset | Block | Concept | Correct | Accuracy | Answer rate | Escalated | Clarify | Avg ms |');
+console.log('| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |');
+for (const row of rows) {
+  console.log(`| ${row.dataset} | ${row.block} | ${row.concept} | ${row.correct}/${row.total} | ${(row.accuracy * 100).toFixed(1)}% | ${(row.answerRate * 100).toFixed(1)}% | ${row.escalated} | ${row.needsClarification} | ${row.avgLatencyMs} |`);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@iola_adm/iola-cli",
-  "version": "0.1.82",
+  "version": "0.1.84",
   "description": "CLI и AI-агент городского округа Йошкар-Ола.",
   "license": "MIT",
   "homepage": "https://github.com/adm-iola/iola-cli#readme",
@@ -24,6 +24,7 @@
     "bin",
     "src",
     "test",
+    "experiments",
     "skills",
     "wiki",
     "docs/assets/readme-header.png",

package/src/cli.js

@@ -6140,7 +6140,7 @@ function pickDirectDataItem(question, dataContext, rows) {
 function itemNameHasNumber(item, number) {
   const name = String(item.name || item.title || item.fns_full_name || item.fns_short_name || "").toLocaleLowerCase("ru-RU");
   const escaped = String(number).replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-  return new RegExp(`(?:№\\s*${escaped}|\\b(?:школа|сош|лицей|гимназия|сад|детский сад)\\s*№?\\s*${escaped}\\b)`, "iu").test(name);
+  return new RegExp(`(?:№\\s*${escaped}(?!\\d)|\\b(?:школа|сош|лицей|гимназия|сад|детский сад)\\s*№?\\s*${escaped}\\b)`, "iu").test(name);
 }
 
 function formatDirectDataField(field, item) {
@@ -6148,7 +6148,7 @@ function formatDirectDataField(field, item) {
   if (field === "head") {
     const head = item.head || item.fns_head_name;
     if (!head) return "";
-    const position = item.fns_head_position || (item.layer === "kindergartens" ? "заведующий" : "директор");
+    const position = capitalizeFirst(item.fns_head_position || (item.layer === "kindergartens" ? "заведующий" : "директор"));
     return `${position}: ${head} (${name}).`;
   }
   if (field === "website") return item.website ? `Сайт: ${item.website}` : `Сайт для ${name} в открытых данных не указан.`;
@@ -6171,6 +6171,11 @@ function getDirectDataItemName(item) {
   return item.name || item.title || item.fns_short_name || item.fns_full_name || "организация";
 }
 
+function capitalizeFirst(value) {
+  const text = String(value || "");
+  return text ? `${text[0].toLocaleUpperCase("ru-RU")}${text.slice(1)}` : text;
+}
+
 async function resolveUsableAiProfile(config, options = {}) {
   const explicit = Boolean(options.profile || options.provider);
   const providerConfig = resolveAiProfile(config, options);
@@ -6589,6 +6594,7 @@ async function buildDataContext(question) {
   try {
     const context = await callPublicMcpTool("layer_answer_context", { question, limit: 8 });
     const layerMap = Object.fromEntries((context.results || []).map((result) => [result.layer?.id || result.layer, result.items || []]));
+    await enrichLayerMapWithExactMatches(layerMap, question, queryTerms, patterns);
     return {
       source: "remote-mcp",
       contract_version: context.contract_version,
@@ -6626,6 +6632,31 @@ async function buildDataContext(question) {
   }
 }
 
+async function enrichLayerMapWithExactMatches(layerMap, question, queryTerms, patterns) {
+  if (!patterns.numbers?.length) return;
+  const targetLayerIds = resolveTargetLayerIds(patterns);
+  await Promise.all(targetLayerIds.map(async (layer) => {
+    try {
+      const result = await queryLayer(layer, { query: question, terms: queryTerms, patterns, limit: 8 });
+      const existing = layerMap[layer] || [];
+      const existingKeys = new Set(existing.map((item) => item.inn || item.name || item.fns_short_name).filter(Boolean));
+      const exact = (result.items || []).filter((item) =>
+        patterns.numbers.some((number) => itemNameHasNumber(item, number)));
+      layerMap[layer] = [
+        ...exact.filter((item) => {
+          const key = item.inn || item.name || item.fns_short_name;
+          if (!key || existingKeys.has(key)) return false;
+          existingKeys.add(key);
+          return true;
+        }),
+        ...existing,
+      ];
+    } catch {
+      // Remote MCP remains the primary source; exact local/API enrichment is best effort.
+    }
+  }));
+}
+
 function resolveTargetLayerIds(patterns = {}) {
   const knownLayers = Object.keys(DATASETS);
   if (patterns.targetLayers?.length) return patterns.targetLayers.filter((layer) => DATASETS[layer]);
@@ -6723,13 +6754,16 @@ function extractSearchTerms(question) {
 
 function extractStructuredPatterns(question) {
   const normalized = question.toLocaleLowerCase("ru-RU");
-  const numbers = [...new Set([...normalized.matchAll(/\b\d{1,3}\b/g)].map((match) => match[0]))];
+  const numbers = [...new Set([
+    ...[...normalized.matchAll(/\b\d{1,3}\b/g)].map((match) => match[0]),
+    ...extractOrdinalNumbers(normalized),
+  ])];
   const inns = [...new Set([...normalized.matchAll(/\b\d{10,12}\b/g)].map((match) => match[0]))];
   const targetLayers = [];
-  if (/(^|[^а-яёa-z])(школа|школы|лицей|лицея|гимназия|гимназии)(?=$|[^а-яёa-z])/iu.test(normalized)) {
+  if (/(школ|сош|лице|гимнази)/iu.test(normalized)) {
     targetLayers.push("schools");
   }
-  if (/(^|[^а-яёa-z])(сад|сады|детсад|детский|детские|доу|мбдоу)(?=$|[^а-яёa-z])/iu.test(normalized)) {
+  if (/(детсад|детск|сад|сады|доу|мбдоу)/iu.test(normalized)) {
     targetLayers.push("kindergartens");
   }
   const streetMatches = [
@@ -6741,6 +6775,24 @@ function extractStructuredPatterns(question) {
   return { numbers, inns, streets, targetLayers: [...new Set(targetLayers)] };
 }
 
+function extractOrdinalNumbers(normalizedQuestion) {
+  const ordinals = [
+    ["1", "(?:перв(?:ая|ой|ую|ое|ого|ом|ым|ых)?|первую)"],
+    ["2", "(?:втор(?:ая|ой|ую|ое|ого|ом|ым|ых)?|вторую)"],
+    ["3", "(?:трет(?:ья|ий|ью|ье|ьего|ьем|ьим|ьих)?|третью)"],
+    ["4", "четверт(?:ая|ой|ую|ое|ого|ом|ым|ых)?"],
+    ["5", "пят(?:ая|ой|ую|ое|ого|ом|ым|ых)?"],
+    ["6", "шест(?:ая|ой|ую|ое|ого|ом|ым|ых)?"],
+    ["7", "седьм(?:ая|ой|ую|ое|ого|ом|ым|ых)?"],
+    ["8", "восьм(?:ая|ой|ую|ое|ого|ом|ым|ых)?"],
+    ["9", "девят(?:ая|ой|ую|ое|ого|ом|ым|ых)?"],
+    ["10", "десят(?:ая|ой|ую|ое|ого|ом|ым|ых)?"],
+  ];
+  return ordinals
+    .filter(([, pattern]) => new RegExp(`(^|[^а-яёa-z])${pattern}(?=$|[^а-яёa-z])`, "iu").test(normalizedQuestion))
+    .map(([number]) => number);
+}
+
 function cleanupPattern(value) {
   return value
     .replace(/\b(школа|школы|сад|детский|детские|сады|лицей|гимназия|контакты|телефон|адрес|найди|покажи)\b/giu, " ")