kushi-agents 6.5.0 → 6.6.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kushi-agents",
-  "version": "6.5.0",
+  "version": "6.6.0",
   "description": "Install Kushi — multi-source project evidence agent with Comprehensive Structured Capture (CSC) into weekly-only files across Email, Teams, OneNote, Loop, SharePoint, Meetings, CRM, ADO. Meetings retain a sibling verbatim/ audit folder. WorkIQ-only for M365 sources (Graph / m365_* FORBIDDEN as fallbacks; user-paste is first-class). Host-agnostic.",
   "type": "module",
   "bin": {

package/plugin/runners/discover.mjs

@@ -21,6 +21,8 @@ import { projectRoot, evidenceRoot, aliasRoot, projectSharedFile, userFile } fro
 import { writeAtomic, pathExists } from './lib/evidence.mjs';
 import { ask as workiqAsk, resolveWorkiqBin } from './lib/workiq.mjs';
 import { loadM365Auth, scopeForSource } from './lib/m365-auth.mjs';
+import { discoverCrm } from './lib/discover-crm.mjs';
+import { discoverAdo } from './lib/discover-ado.mjs';
 
 const ALL_SOURCES = ['email', 'teams', 'meetings', 'onenote', 'sharepoint', 'crm', 'ado'];
 
@@ -407,6 +409,76 @@ async function main() {
       sourceResults.push({ source, asked: false, found: 0, accepted: [], skipped_reason: 'disabled-in-config' });
       continue;
     }
+    // CRM and ADO are NEVER WorkIQ — they use mapped REST endpoints + fuzzy match
+    // (per fuzzy-disambiguation.instructions.md, v4.4.7+). Handle them inline.
+    if (source === 'crm' || source === 'ado') {
+      const projectName = path.basename(root);
+      const t0r = Date.now();
+      log(`[${idx}/${total}] ${source}: REST fuzzy-match against ${source === 'crm' ? 'Dataverse' : 'Azure DevOps'} (mapped settings)`);
+      const result = source === 'crm'
+        ? await discoverCrm(projectName, integ.crm, {})
+        : await discoverAdo(projectName, integ.ado, {});
+      const elapsed = Date.now() - t0r;
+      // Persist raw result so users can inspect candidates that didn't auto-pick.
+      if (!args.dryRun) {
+        try {
+          const discoveryDir = path.join(aliasRoot(args.project, args.alias), '_discovery');
+          await fs.mkdir(discoveryDir, { recursive: true });
+          const header = `# discover ${source} @ ${new Date().toISOString()}\n# elapsed=${elapsed}ms decision=${result.decision} confidence=${result.confidence || ''} candidates=${result.candidates?.length || 0}${result.error ? ' error=' + result.error : ''}\n`;
+          await fs.writeFile(
+            path.join(discoveryDir, `${source}-raw.txt`),
+            header + JSON.stringify({ picked: result.picked, candidates: result.candidates, error: result.error }, null, 2),
+            'utf8',
+          );
+        } catch { /* best-effort */ }
+      }
+      let accepted = [];
+      let unresolved = null;
+      let integrationsPatch = null;
+      if (result.decision === 'auto' && result.picked) {
+        if (source === 'crm') {
+          const cur = integ.crm || {};
+          const id = result.picked.id;
+          integrationsPatch = { crm: { ...cur, request_id: id } };
+          accepted.push(id);
+          log(`  crm: ✓ resolved → "${result.picked.title}" (${id}) [score=${result.picked.score}, confidence=${result.confidence}]`);
+        } else {
+          const cur = integ.ado || {};
+          const id = result.picked.id;
+          integrationsPatch = { ado: { ...cur, engagement_id: id } };
+          accepted.push(id);
+          log(`  ado: ✓ resolved → "${result.picked.title}" (${id}) [score=${result.picked.score}, confidence=${result.confidence}]`);
+        }
+      } else {
+        unresolved = source === 'crm' ? 'crm.request_id' : 'ado.engagement_id';
+        if (result.error) {
+          log(`  ${source}: ✗ ${result.error}`);
+        } else if (result.candidates?.length) {
+          log(`  ${source}: ⚠ ${result.candidates.length} candidate(s) but none confident enough — see _discovery/${source}-raw.txt`);
+          for (const c of result.candidates.slice(0, 5)) {
+            log(`    • ${c.id} · "${c.title}" [score=${c.score}, kind=${c.kind}]`);
+          }
+        } else {
+          log(`  ${source}: ⚠ no candidates matched project name`);
+        }
+      }
+      if (integrationsPatch) {
+        Object.assign(integ, mergeShallow(integ, integrationsPatch));
+        integDirty = true;
+      }
+      sourceResults.push({
+        source,
+        asked: true,
+        found: result.candidates?.length || 0,
+        accepted,
+        unresolved,
+        candidates: result.candidates?.slice(0, 5) || [],
+        error: result.error,
+        skipped_reason: result.error ? 'rest-error' : (unresolved ? 'no-confident-match' : null),
+      });
+      continue;
+    }
+
     const prompt = buildPrompt(source, path.basename(root), scope);
     let rows = [];
     let asked = true;

package/plugin/runners/lib/discover-ado.mjs

@@ -0,0 +1,73 @@
+// plugin/runners/lib/discover-ado.mjs
+// Direct ADO WIQL fuzzy search for project -> Engagement work item.
+// Replaces WorkIQ for ado discovery — uses mapped org/project from integrations.yml.
+
+import { fetchWithRetry } from './http.mjs';
+import { authHeader, SCOPES } from './identity.mjs';
+import { rank, decide } from './fuzzy-match.mjs';
+
+/**
+ * @param {string} projectName
+ * @param {{organization?:string, project?:string, apiVersion?:string, workItemType?:string}} adoConfig
+ * @param {{fetcher?:Function, headersOverride?:object}} [opts]
+ */
+export async function discoverAdo(projectName, adoConfig, opts = {}) {
+  const org = adoConfig?.organization;
+  const project = adoConfig?.project;
+  const apiVersion = adoConfig?.apiVersion || '7.1';
+  const witType = adoConfig?.workItemTypes?.engagement || 'Engagement';
+  if (!org || !project || /<.*>/.test(org) || /<.*>/.test(project)) {
+    return { decision: 'unresolved', candidates: [], error: 'ado organization/project not configured', source: 'ado' };
+  }
+  const wiql = `SELECT [System.Id],[System.Title],[System.WorkItemType] FROM workitems WHERE [System.TeamProject]='${escapeWiql(project)}' AND [System.WorkItemType]='${escapeWiql(witType)}' AND [System.Title] CONTAINS '${escapeWiql(projectName)}'`;
+  const url = `https://dev.azure.com/${encodeURIComponent(org)}/${encodeURIComponent(project)}/_apis/wit/wiql?api-version=${apiVersion}`;
+  let headers;
+  try {
+    headers = opts.headersOverride || await authHeader(SCOPES.ado, { 'Content-Type': 'application/json' });
+  } catch (e) {
+    return { decision: 'unresolved', candidates: [], error: `ado auth failed: ${e.message}`, source: 'ado' };
+  }
+  let res;
+  try {
+    res = await fetchWithRetry(url, {
+      method: 'POST',
+      headers,
+      body: JSON.stringify({ query: wiql }),
+      fetcher: opts.fetcher,
+      retries: 2,
+    });
+  } catch (e) {
+    return { decision: 'unresolved', candidates: [], error: `ado wiql fetch failed: ${e.message}`, source: 'ado' };
+  }
+  if (!res.ok) {
+    const body = await safeText(res);
+    return { decision: 'unresolved', candidates: [], error: `ado http ${res.status}: ${body.slice(0, 200)}`, source: 'ado' };
+  }
+  const wiqlData = await res.json().catch(() => ({}));
+  const ids = (wiqlData.workItems || []).map(w => w.id).filter(Boolean).slice(0, 50);
+  if (!ids.length) return { decision: 'unresolved', candidates: [], source: 'ado' };
+
+  // Hydrate titles via /workitems?ids=...
+  const detailsUrl = `https://dev.azure.com/${encodeURIComponent(org)}/${encodeURIComponent(project)}/_apis/wit/workitems?ids=${ids.join(',')}&fields=System.Id,System.Title,System.WorkItemType,System.State&api-version=${apiVersion}`;
+  let dres;
+  try {
+    dres = await fetchWithRetry(detailsUrl, { headers, fetcher: opts.fetcher, retries: 2 });
+  } catch (e) {
+    return { decision: 'unresolved', candidates: [], error: `ado workitems fetch failed: ${e.message}`, source: 'ado' };
+  }
+  if (!dres.ok) {
+    const body = await safeText(dres);
+    return { decision: 'unresolved', candidates: [], error: `ado workitems http ${dres.status}: ${body.slice(0, 200)}`, source: 'ado' };
+  }
+  const detData = await dres.json().catch(() => ({}));
+  const candidates = (detData.value || []).map(w => ({
+    id: String(w.id),
+    title: w.fields?.['System.Title'] || '',
+    secondary: `${w.fields?.['System.WorkItemType'] || ''} · ${w.fields?.['System.State'] || ''}`,
+  })).filter(c => c.title);
+  const ranked = rank(projectName, candidates);
+  return { ...decide(ranked), source: 'ado' };
+}
+
+function escapeWiql(v) { return String(v).replace(/'/g, "''"); }
+async function safeText(res) { try { return await res.text(); } catch { return ''; } }

package/plugin/runners/lib/discover-crm.mjs

@@ -0,0 +1,60 @@
+// plugin/runners/lib/discover-crm.mjs
+// Direct Dataverse OData fuzzy search for project -> CRM record.
+// Replaces WorkIQ for crm discovery — uses mapped instance from integrations.yml.
+
+import { fetchWithRetry, encodeODataOp } from './http.mjs';
+import { authHeader } from './identity.mjs';
+import { SCOPES } from './identity.mjs';
+import { rank, decide } from './fuzzy-match.mjs';
+
+/**
+ * @param {string} projectName
+ * @param {{instance?:string, table?:string}} integ.crm
+ * @param {{fetcher?:Function, headersOverride?:object}} [opts]
+ * @returns {Promise<{decision, picked?, candidates, error?, source:'crm'}>}
+ */
+export async function discoverCrm(projectName, crmConfig, opts = {}) {
+  const instance = (crmConfig?.instance || '').replace(/\/$/, '');
+  if (!instance || /<.*>/.test(instance)) {
+    return { decision: 'unresolved', candidates: [], error: 'crm.instance not configured', source: 'crm' };
+  }
+  const table = crmConfig?.table || 'incidents';
+  const titleField = table === 'incidents' ? 'title' : 'name';
+  const filter = `contains(${titleField},'${escapeOData(projectName)}') or contains(description,'${escapeOData(projectName)}')`;
+  const select = table === 'incidents'
+    ? 'incidentid,title,ticketnumber,description'
+    : `${table}id,${titleField}`;
+  const url = `${instance}/api/data/v9.2/${table}?${encodeODataOp('$filter')}=${encodeURIComponent(filter)}&${encodeODataOp('$select')}=${encodeURIComponent(select)}&${encodeODataOp('$top')}=20`;
+  let headers;
+  try {
+    headers = opts.headersOverride || await authHeader(SCOPES.dataverse(instance), {
+      'OData-MaxVersion': '4.0',
+      'OData-Version': '4.0',
+      Prefer: 'odata.include-annotations="*"',
+    });
+  } catch (e) {
+    return { decision: 'unresolved', candidates: [], error: `crm auth failed: ${e.message}`, source: 'crm' };
+  }
+  let res;
+  try {
+    res = await fetchWithRetry(url, { headers, fetcher: opts.fetcher, retries: 2 });
+  } catch (e) {
+    return { decision: 'unresolved', candidates: [], error: `crm fetch failed: ${e.message}`, source: 'crm' };
+  }
+  if (!res.ok) {
+    const body = await safeText(res);
+    return { decision: 'unresolved', candidates: [], error: `crm http ${res.status}: ${body.slice(0, 200)}`, source: 'crm' };
+  }
+  const data = await res.json().catch(() => ({}));
+  const rows = Array.isArray(data.value) ? data.value : [];
+  const candidates = rows.map(r => ({
+    id: r.ticketnumber || r.incidentid || r[`${table}id`] || '',
+    title: r.title || r[titleField] || '',
+    secondary: r.ticketnumber ? `incidentid=${r.incidentid}` : '',
+  })).filter(c => c.id && c.title);
+  const ranked = rank(projectName, candidates);
+  return { ...decide(ranked), source: 'crm' };
+}
+
+function escapeOData(v) { return String(v).replace(/'/g, "''"); }
+async function safeText(res) { try { return await res.text(); } catch { return ''; } }

package/plugin/runners/lib/fuzzy-match.mjs

@@ -0,0 +1,95 @@
+// plugin/runners/lib/fuzzy-match.mjs
+// Universal fuzzy disambiguation per
+// plugin/instructions/fuzzy-disambiguation.instructions.md (v4.4.7+).
+//
+// Score lexicographic order:
+//   exact (case-insensitive) ........... 1000
+//   prefix (target startsWith query) ... 800 - max(0, target.length - query.length)
+//   contains (target includes query) ... 500 - distance-from-start
+//   token-overlap (>=1 shared token) ... 300 + matchedTokens * 10
+//   levenshtein (<=2 edits) ............ 100 + (3 - distance) * 10
+//   else ............................... 0
+//
+// Decision rule (top 5):
+//   top >= 800 AND #2 <= top * 0.7  -> auto-pick      (confidence: 'high')
+//   top >= 300 AND #2 <= top * 0.7  -> auto-pick warn (confidence: 'medium')
+//   else                            -> unresolved + return ranked candidates
+
+const norm = (s) => String(s ?? '').trim().toLowerCase();
+
+function levenshtein(a, b) {
+  if (a === b) return 0;
+  const m = a.length, n = b.length;
+  if (!m) return n;
+  if (!n) return m;
+  if (Math.abs(m - n) > 2) return Infinity; // we only care about <=2
+  const prev = new Array(n + 1);
+  const curr = new Array(n + 1);
+  for (let j = 0; j <= n; j++) prev[j] = j;
+  for (let i = 1; i <= m; i++) {
+    curr[0] = i;
+    let row = i;
+    for (let j = 1; j <= n; j++) {
+      const cost = a.charCodeAt(i - 1) === b.charCodeAt(j - 1) ? 0 : 1;
+      curr[j] = Math.min(curr[j - 1] + 1, prev[j] + 1, prev[j - 1] + cost);
+      if (curr[j] < row) row = curr[j];
+    }
+    if (row > 2) return Infinity; // early-exit
+    for (let j = 0; j <= n; j++) prev[j] = curr[j];
+  }
+  return prev[n];
+}
+
+/**
+ * Score a single candidate name against a query.
+ * @param {string} query
+ * @param {string} target
+ * @returns {{score:number, kind:string}}
+ */
+export function scoreOne(query, target) {
+  const q = norm(query);
+  const t = norm(target);
+  if (!q || !t) return { score: 0, kind: 'empty' };
+  if (q === t) return { score: 1000, kind: 'exact' };
+  if (t.startsWith(q)) return { score: 800 - Math.max(0, t.length - q.length), kind: 'prefix' };
+  const idx = t.indexOf(q);
+  if (idx >= 0) return { score: 500 - idx, kind: 'contains' };
+  const qTokens = q.split(/\s+/).filter(Boolean);
+  const tTokens = t.split(/\s+/).filter(Boolean);
+  const shared = qTokens.filter(x => tTokens.includes(x)).length;
+  if (shared >= 1) return { score: 300 + shared * 10, kind: 'token-overlap' };
+  const d = levenshtein(q, t);
+  if (d <= 2) return { score: 100 + (3 - d) * 10, kind: `levenshtein-${d}` };
+  return { score: 0, kind: 'none' };
+}
+
+/**
+ * Score and rank an array of candidates.
+ * @param {string} query
+ * @param {Array<{id:string, title:string, [k:string]:any}>} candidates
+ * @returns {Array} sorted desc by score (input shape preserved + score + kind)
+ */
+export function rank(query, candidates) {
+  const out = [];
+  for (const c of candidates) {
+    const { score, kind } = scoreOne(query, c.title || c.name || '');
+    if (score > 0) out.push({ ...c, score, kind });
+  }
+  out.sort((a, b) => b.score - a.score);
+  return out;
+}
+
+/**
+ * Apply the v4.4.7 decision rule.
+ * @returns {{decision:'auto'|'unresolved', confidence?:string, picked?:object, candidates:Array}}
+ */
+export function decide(ranked) {
+  const top5 = ranked.slice(0, 5);
+  if (!top5.length) return { decision: 'unresolved', candidates: [] };
+  const top = top5[0];
+  const second = top5[1];
+  const gapOk = !second || second.score <= top.score * 0.7;
+  if (top.score >= 800 && gapOk) return { decision: 'auto', confidence: 'high', picked: top, candidates: top5 };
+  if (top.score >= 300 && gapOk) return { decision: 'auto', confidence: 'medium', picked: top, candidates: top5 };
+  return { decision: 'unresolved', candidates: top5 };
+}

package/plugin/runners/test/unit/discover-rest.test.mjs

@@ -0,0 +1,81 @@
+// plugin/runners/test/unit/discover-rest.test.mjs
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { discoverCrm } from '../../lib/discover-crm.mjs';
+import { discoverAdo } from '../../lib/discover-ado.mjs';
+
+function mockResponse({ status = 200, json = {} } = {}) {
+  return {
+    ok: status >= 200 && status < 300,
+    status,
+    statusText: 'mock',
+    headers: { get: () => null },
+    json: async () => json,
+    text: async () => JSON.stringify(json),
+  };
+}
+
+test('discoverCrm: returns unresolved when instance not configured', async () => {
+  const r = await discoverCrm('HCA', { instance: '<__FILL_ME_IN__>' });
+  assert.equal(r.decision, 'unresolved');
+  assert.match(r.error, /not configured/);
+});
+
+test('discoverCrm: auto-picks high-confidence match', async () => {
+  const fetcher = async () => mockResponse({
+    json: { value: [
+      { incidentid: 'g-1', title: 'HCA Agentic Engineering', ticketnumber: 'FE-2026-001458' },
+      { incidentid: 'g-2', title: 'unrelated', ticketnumber: 'FE-2025-000001' },
+    ] },
+  });
+  const r = await discoverCrm('HCA Agentic Engineering', { instance: 'https://iscrm.crm.dynamics.com' }, {
+    fetcher, headersOverride: { Authorization: 'Bearer fake' },
+  });
+  assert.equal(r.decision, 'auto');
+  assert.equal(r.picked.id, 'FE-2026-001458');
+});
+
+test('discoverCrm: returns unresolved on 401', async () => {
+  const fetcher = async () => mockResponse({ status: 401, json: { error: 'unauth' } });
+  const r = await discoverCrm('HCA', { instance: 'https://iscrm.crm.dynamics.com' }, {
+    fetcher, headersOverride: { Authorization: 'Bearer fake' },
+  });
+  assert.equal(r.decision, 'unresolved');
+  assert.match(r.error, /401/i);
+});
+
+test('discoverAdo: returns unresolved when org not configured', async () => {
+  const r = await discoverAdo('HCA', { organization: '<x>', project: 'IS Engagements' });
+  assert.equal(r.decision, 'unresolved');
+  assert.match(r.error, /not configured/);
+});
+
+test('discoverAdo: auto-picks from WIQL + workitems hydrate', async () => {
+  let call = 0;
+  const fetcher = async (url) => {
+    call++;
+    if (call === 1) {
+      // WIQL POST -> ids only
+      return mockResponse({ json: { workItems: [{ id: 12345 }, { id: 67890 }] } });
+    }
+    // hydrate
+    return mockResponse({ json: { value: [
+      { id: 12345, fields: { 'System.Title': 'HCA Engagement', 'System.WorkItemType': 'Engagement', 'System.State': 'Active' } },
+      { id: 67890, fields: { 'System.Title': 'Different Thing', 'System.WorkItemType': 'Engagement', 'System.State': 'Closed' } },
+    ] } });
+  };
+  const r = await discoverAdo('HCA', {
+    organization: 'IndustrySolutions', project: 'IS Engagements', apiVersion: '7.1',
+  }, { fetcher, headersOverride: { Authorization: 'Bearer fake', 'Content-Type': 'application/json' } });
+  assert.equal(r.decision, 'auto');
+  assert.equal(r.picked.id, '12345');
+});
+
+test('discoverAdo: empty WIQL result -> unresolved', async () => {
+  const fetcher = async () => mockResponse({ json: { workItems: [] } });
+  const r = await discoverAdo('HCA', {
+    organization: 'IndustrySolutions', project: 'IS Engagements',
+  }, { fetcher, headersOverride: { Authorization: 'Bearer fake', 'Content-Type': 'application/json' } });
+  assert.equal(r.decision, 'unresolved');
+  assert.equal(r.candidates.length, 0);
+});

package/plugin/runners/test/unit/fuzzy-match.test.mjs

@@ -0,0 +1,82 @@
+// plugin/runners/test/unit/fuzzy-match.test.mjs
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { scoreOne, rank, decide } from '../../lib/fuzzy-match.mjs';
+
+test('scoreOne: exact match scores 1000', () => {
+  assert.equal(scoreOne('HCA', 'hca').score, 1000);
+});
+
+test('scoreOne: prefix beats contains', () => {
+  const p = scoreOne('HCA', 'HCA Agentic Engineering').score;
+  const c = scoreOne('Agentic', 'HCA Agentic Engineering').score;
+  assert.ok(p > c, `prefix=${p} should be > contains=${c}`);
+});
+
+test('scoreOne: contains scores below prefix but above token-overlap', () => {
+  assert.ok(scoreOne('foo', 'bar foo baz').score >= 300);
+});
+
+test('scoreOne: token-overlap when no substring match', () => {
+  const r = scoreOne('northwind acme', 'acme corp engagement');
+  assert.ok(r.score >= 300 && r.score < 500, `score=${r.score} kind=${r.kind}`);
+});
+
+test('scoreOne: levenshtein within 2 edits', () => {
+  const r = scoreOne('hca', 'hsa');
+  assert.ok(r.score >= 100 && r.score <= 130, `score=${r.score}`);
+});
+
+test('scoreOne: zero on no relation', () => {
+  assert.equal(scoreOne('aaaa', 'zzzzzzzz').score, 0);
+});
+
+test('rank: drops zero scores and sorts desc', () => {
+  const out = rank('HCA', [
+    { id: '1', title: 'unrelated thing' },
+    { id: '2', title: 'HCA Agentic Engineering' },
+    { id: '3', title: 'hca' },
+  ]);
+  assert.equal(out.length, 2);
+  assert.equal(out[0].id, '3'); // exact
+  assert.equal(out[1].id, '2'); // prefix
+});
+
+test('decide: auto-pick high when top exact and gap big', () => {
+  const ranked = rank('HCA', [
+    { id: 'A', title: 'hca' },
+    { id: 'B', title: 'something else' },
+  ]);
+  const d = decide(ranked);
+  assert.equal(d.decision, 'auto');
+  assert.equal(d.confidence, 'high');
+  assert.equal(d.picked.id, 'A');
+});
+
+test('decide: unresolved when two strong candidates close', () => {
+  const ranked = rank('HCA', [
+    { id: 'A', title: 'HCA West' },
+    { id: 'B', title: 'HCA East' },
+  ]);
+  const d = decide(ranked);
+  // both are prefix-scored equally (or close) -> gap fails -> unresolved
+  assert.equal(d.decision, 'unresolved');
+  assert.equal(d.candidates.length, 2);
+});
+
+test('decide: unresolved when no candidates', () => {
+  const d = decide([]);
+  assert.equal(d.decision, 'unresolved');
+  assert.equal(d.candidates.length, 0);
+});
+
+test('decide: medium-confidence auto for token-overlap with clear gap', () => {
+  const ranked = rank('northwind project', [
+    { id: 'A', title: 'northwind delivery' }, // token-overlap on northwind
+    { id: 'B', title: 'totally different thing' }, // 0
+  ]);
+  const d = decide(ranked);
+  assert.equal(d.decision, 'auto');
+  assert.equal(d.confidence, 'medium');
+  assert.equal(d.picked.id, 'A');
+});