kushi-agents 6.1.0 → 6.1.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kushi-agents",
-  "version": "6.1.0",
+  "version": "6.1.2",
   "description": "Install Kushi — multi-source project evidence agent with Comprehensive Structured Capture (CSC) into weekly-only files across Email, Teams, OneNote, Loop, SharePoint, Meetings, CRM, ADO. Meetings retain a sibling verbatim/ audit folder. WorkIQ-only for M365 sources (Graph / m365_* FORBIDDEN as fallbacks; user-paste is first-class). Host-agnostic.",
   "type": "module",
   "bin": {

package/plugin/instructions/evidence-layout-canonical.instructions.md

@@ -62,8 +62,8 @@ Concretely, a pull skill MUST NOT create or write to any of these (representativ
 | `<project>/_Weekly Summaries/` | `<project>/Evidence/<alias>/<source>/weekly/` |
 | `<project>/Meetings/` (at project root) | `<project>/Evidence/<alias>/meetings/` |
 | `<project>/Teams/` (at project root) | `<project>/Evidence/<alias>/teams/` |
-| `<project>/SharePoint/` (at project root) | `<project>/Evidence/<alias>/sharepoint/` |
-| `<project>/CRM/`, `<project>/ADO/` (at project root) | `<project>/Evidence/<alias>/{crm,ado}/` |
+| `<project>/SharePoint/` (at project root) | `<project>/Evidence/_shared/sharepoint/` (project-scoped; v6.1.2+) |
+| `<project>/CRM/`, `<project>/ADO/` (at project root) | `<project>/Evidence/_shared/{crm,ado}/` |
 | any `<project>/<source>-context/`, `<project>/<source>-summary/`, etc. | `<project>/Evidence/<alias>/<source>/weekly/` |
 
 `State/`, `Reports/`, `integrations.yml` are the **only** top-level siblings of `Evidence/` that pull/refresh skills are allowed to leave alone (they are written by `build-state`, `aggregate-project`, and bootstrap respectively — not by pull-* skills).
@@ -83,7 +83,7 @@ If alias is ambiguous (legacy folder pre-dates multi-user), default to the curre
 
 By contract, downstream skills walk:
 
-- `Evidence/*/email/`, `Evidence/*/teams/`, `Evidence/*/meetings/`, `Evidence/*/onenote/`, `Evidence/*/sharepoint/`, `Evidence/*/crm/`, `Evidence/*/ado/`
+- `Evidence/*/email/`, `Evidence/*/teams/`, `Evidence/*/meetings/`, `Evidence/*/onenote/`, `Evidence/_shared/sharepoint/`, `Evidence/_shared/crm/`, `Evidence/_shared/ado/`
 
 Anything outside these paths is invisible to them — by design. The path IS the contract. There is no "also scan sibling folders" fallback.
 

package/plugin/runners/bootstrap.mjs

@@ -26,6 +26,7 @@ import {
   aliasRoot, projectSharedFile, userFile, USER_FILES,
 } from './lib/layout.mjs';
 import { writeAtomic, pathExists } from './lib/evidence.mjs';
+import { writeRefreshReport, appendRunLog } from './lib/runlog.mjs';
 
 function parseArgs(argv) {
   const args = { force: false, dryRun: false, lookbackDays: null, interactive: false };
@@ -254,7 +255,7 @@ const USER_DIRS = [
   USER_FILES.discovery,
   USER_FILES.deferredRetries,
   USER_FILES.refreshReports,
-  'email', 'teams', 'meetings', 'onenote', 'sharepoint',
+  'email', 'teams', 'meetings', 'onenote',
   'crm-notes', 'ado-notes',
 ];
 
@@ -327,6 +328,63 @@ async function main() {
     });
   }
 
+  const startedAt = new Date(); // capture when scaffold started (before report)
+  // v6.1.2: write bootstrap report to refresh-reports/<ts>_bootstrap.md per
+  // run-reports doctrine. Bootstrap is scaffold-only (no HTTP, no pulls), so
+  // the report enumerates files/dirs created vs. existed, plus optional
+  // dateFloor + interactive outcomes. Diagnostics-only; never blocks.
+  let reportPath = null;
+  if (!args.dryRun) {
+    try {
+      const created = log.created.map(p => path.relative(root, p) || '.');
+      const existed = log.existed.map(p => path.relative(root, p) || '.');
+      const summaryLine = `bootstrap ${args.alias}: created=${created.length} existed=${existed.length}` +
+        (dateFloorReport && dateFloorReport.fields?.length ? ` dateFloor=${dateFloorReport.dateFloor}` : '') +
+        (interactiveReport && interactiveReport.fields?.length ? ` interactive=${interactiveReport.fields.length}` : '');
+      const details = {
+        mode: 'bootstrap',
+        contributor: args.alias,
+        started: startedAt.toISOString(),
+        ended: new Date().toISOString(),
+        scope: 'scaffold-only (no HTTP, no pulls)',
+        what_was_done: {
+          configs_probed: ['integrations.yml', 'project-info.md', 'external-links.yml', 'contributors.yml'],
+          shared_dirs: SHARED_DIRS,
+          user_dirs: USER_DIRS,
+          per_user_files: ['boundaries.yml', 'external-links.local.yml', '_ledger.yml'],
+          counts: {
+            created: created.length,
+            existed: existed.length,
+          },
+        },
+        created,
+        existed,
+        date_floor: dateFloorReport || null,
+        interactive: interactiveReport || null,
+        skips_and_gaps: [
+          'Bootstrap is scaffold-only — no source pulls performed.',
+          'Run `kushi discover <project>` next to populate boundaries.yml/integrations.yml.',
+          'Then `kushi refresh <project>` to capture per-source CSC weekly files.',
+        ],
+      };
+      const r = await writeRefreshReport(args.project, args.alias, {
+        type: 'bootstrap',
+        summary: summaryLine,
+        details,
+      });
+      reportPath = r?.path || null;
+      try {
+        await appendRunLog(args.project, {
+          mode: 'bootstrap',
+          contributor: args.alias,
+          status: 'ok',
+          summary: summaryLine,
+          report: reportPath ? path.relative(root, reportPath) : null,
+        });
+      } catch { /* run-log is diagnostics-only */ }
+    } catch { /* bootstrap-report is diagnostics-only, never block */ }
+  }
+
   emit({
     status: 'ok',
     project: root,
@@ -334,6 +392,7 @@ async function main() {
     created: log.created.map(p => path.relative(root, p) || '.'),
     existed: log.existed.map(p => path.relative(root, p) || '.'),
     dry_run: args.dryRun,
+    ...(reportPath ? { report: path.relative(root, reportPath) } : {}),
     ...(dateFloorReport ? { date_floor: dateFloorReport } : {}),
     ...(interactiveReport ? { interactive: interactiveReport } : {}),
   });

package/plugin/runners/discover.mjs

@@ -66,8 +66,8 @@ function buildPrompt(source, projectName, scope = null) {
     meetings: `Find recurring Teams meeting series related to project "${projectName}".`,
     onenote: `Find OneNote sections related to project "${projectName}".`,
     sharepoint: `Find SharePoint sites or libraries related to project "${projectName}".`,
-    crm: `Find Dataverse CRM incidents/requests related to project "${projectName}" in iscrm.crm.dynamics.com.`,
-    ado: `Find Azure DevOps engagement work items in IndustrySolutions/IS Engagements related to project "${projectName}".`,
+    crm: `Search the user's indexed M365 content (emails, OneNote pages, SharePoint/OneDrive documents, Teams messages) for Dataverse CRM request/incident identifiers that relate to project "${projectName}". CRM IDs typically look like FE-2026-001458, REQ-12345, INC-9, or 6+ digit numerics, and appear in email signatures/subjects, project briefs, status decks, or Loop pages from iscrm.crm.dynamics.com. Return the request_id (preferred) or incident_number you see most consistently associated with the project.`,
+    ado: `Search the user's indexed M365 content (emails, OneNote pages, SharePoint/OneDrive documents, Teams messages) for Azure DevOps work items related to project "${projectName}" in the IndustrySolutions/IS Engagements org. Engagement work items are typically referenced by 5–7 digit numeric IDs in URLs like dev.azure.com/IndustrySolutions/IS%20Engagements/_workitems/edit/123456 or in shorthand like "AB#123456" / "WI 123456". Return the engagement_id (top-level "Engagement" work item id, preferred) or work_item_id you see most consistently associated with the project.`,
   };
   const fields = {
     email: 'value (full folder path like "Inbox/Northwind"), confidence (high|medium|low)',
@@ -162,7 +162,7 @@ function isPlaceholder(v) {
   if (!s) return true;
   if (/^<.*>$/.test(s)) return true;              // <value>, <chat_id>, etc.
   if (/^turn\d+search\d+$/i.test(s)) return true; // WorkIQ web citation tokens
-  if (/^(unknown|n\/a|none|null|tbd|todo)$/i.test(s)) return true;
+  if (/^(unknown|n\/a|none|null|tbd|todo|not\s+explicitly\s+(available|provided|specified)|not\s+(available|provided|specified|applicable)|undisclosed)$/i.test(s)) return true;
   return false;
 }
 
@@ -178,19 +178,26 @@ function isValidValueFor(source, field, raw) {
     return true;
   }
   if (source === 'teams' && field === 'chat_id') {
-    // Graph chat IDs are long strings ending in '@thread.v2' or similar.
-    return v.includes('@thread') || v.length > 30;
+    // Accept Graph chat IDs ('@thread' / long opaque) OR human-readable
+    // chat topics — WorkIQ frequently returns the topic when the Graph ID
+    // is unavailable. Refresh stage will resolve to a real chat.
+    return v.length >= 2 && v.length <= 200;
   }
   if (source === 'meetings' && field === 'join_url') {
-    return /^https:\/\/teams\.microsoft\.com\/.+meetup-join/i.test(v);
+    // Prefer real Teams meetup URLs but also accept any https URL or
+    // a meeting subject — refresh stage will resolve.
+    if (v.startsWith('http')) return true;
+    return v.length >= 3 && v.length <= 200;
   }
   if (source === 'onenote' && field === 'section_file_id') {
-    // OneNote section file IDs are hex strings, typically prefixed `0-` and
-    // dozens of chars long. Reject short citation tokens.
-    return /^[0-9a-f][0-9a-f\-]{20,}$/i.test(v) || v.startsWith('0-');
+    // Accept hex section IDs OR section/page names. WorkIQ often only has
+    // the human-readable name; refresh resolves to the file ID.
+    return v.length >= 2 && v.length <= 200;
   }
   if (source === 'sharepoint' && field === 'site_url') {
-    return v.startsWith('https://') && v.includes('.sharepoint.com');
+    // Prefer real SharePoint URLs; also accept site names/relative paths.
+    if (v.startsWith('http')) return v.includes('.sharepoint.com') || v.includes('/sites/');
+    return v.length >= 2 && v.length <= 300;
   }
   if (source === 'crm' && (field === 'request_id' || field === 'incident_number')) {
     // CRM IDs vary: FE-2026-001458, REQ-12345, INC-9, plain numerics. Reject
@@ -228,7 +235,16 @@ function applyRows(source, rows, currentBounds, currentInteg) {
   }
   if (source === 'teams') {
     const existing = currentBounds.teams?.chats || [];
-    const incoming = rows.map(r => r.chat_id).filter(v => isValidValueFor('teams', 'chat_id', v));
+    // WorkIQ rarely has Graph chat IDs — citation tokens like "turn1search1"
+    // are common. Prefer chat_id; fall back to topic so refresh has a usable
+    // boundary descriptor instead of nothing.
+    const incoming = rows.map(r => {
+      const id = r.chat_id;
+      if (id && !isPlaceholder(id) && isValidValueFor('teams', 'chat_id', id)) return id;
+      const topic = r.topic;
+      if (topic && !isPlaceholder(topic)) return topic;
+      return null;
+    }).filter(Boolean);
     const merged = dedup([...existing, ...incoming]);
     const added = merged.filter(v => !existing.includes(v));
     if (added.length) accepted.push(...added);
@@ -236,7 +252,13 @@ function applyRows(source, rows, currentBounds, currentInteg) {
   }
   if (source === 'meetings') {
     const existing = currentBounds.meetings?.joinUrls || [];
-    const incoming = rows.map(r => r.join_url).filter(v => isValidValueFor('meetings', 'join_url', v));
+    const incoming = rows.map(r => {
+      const url = r.join_url;
+      if (url && !isPlaceholder(url) && isValidValueFor('meetings', 'join_url', url) && url.startsWith('http')) return url;
+      const subj = r.subject;
+      if (subj && !isPlaceholder(subj)) return subj;
+      return null;
+    }).filter(Boolean);
     const merged = dedup([...existing, ...incoming]);
     const added = merged.filter(v => !existing.includes(v));
     if (added.length) accepted.push(...added);
@@ -244,7 +266,16 @@ function applyRows(source, rows, currentBounds, currentInteg) {
   }
   if (source === 'onenote') {
     const existing = currentBounds.onenote?.section_file_ids || [];
-    const incoming = rows.map(r => r.section_file_id).filter(v => isValidValueFor('onenote', 'section_file_id', v));
+    // WorkIQ rarely has section_file_id (Graph property); usually returns
+    // citation tokens. Prefer real hex IDs; fall back to section_name so
+    // refresh has a usable boundary descriptor.
+    const incoming = rows.map(r => {
+      const id = r.section_file_id;
+      if (id && !isPlaceholder(id) && /^[0-9a-f][0-9a-f\-]{20,}$/i.test(String(id).trim())) return id;
+      const name = r.section_name;
+      if (name && !isPlaceholder(name)) return name;
+      return null;
+    }).filter(Boolean);
     const merged = dedup([...existing, ...incoming]);
     const added = merged.filter(v => !existing.includes(v));
     if (added.length) accepted.push(...added);
@@ -391,6 +422,16 @@ async function main() {
       rows = rowsFromBlocks(blocks, source);
       const elapsed = Date.now() - t0;
       const bytes = Buffer.byteLength(workiqStdout || '', 'utf8');
+      // Persist raw WorkIQ output for diagnosis. Lets users inspect why a
+      // source returned 0 accepted rows without re-running the query.
+      if (!args.dryRun) {
+        try {
+          const discoveryDir = path.join(aliasRoot(args.project, args.alias), '_discovery');
+          await fs.mkdir(discoveryDir, { recursive: true });
+          const header = `# discover ${source} @ ${new Date().toISOString()}\n# elapsed=${elapsed}ms bytes=${bytes} blocks=${blocks.length} rows=${rows.length}\n# prompt:\n${prompt.split('\n').map(l => '# ' + l).join('\n')}\n# --- workiq stdout ---\n`;
+          await fs.writeFile(path.join(discoveryDir, `${source}-raw.txt`), header + (workiqStdout || ''), 'utf8');
+        } catch { /* best-effort */ }
+      }
       if (rows.length === 0 && bytes < 8) {
         // Distinguish "workiq returned empty" from "timeout" — both used to look the same.
         skipReason = 'workiq-empty-response';

package/plugin/runners/lib/csc-pull.mjs

@@ -69,8 +69,10 @@ export function buildPullPrompt({ source, project, entity, weekStart, scope, opt
     if (scope?.notebookName) lines.push(`Restrict to notebook "${scope.notebookName}".`);
     lines.push('One CSC block per page touched in the week.');
   } else if (source === 'sharepoint') {
-    lines.push(`Find SharePoint files within site "${entity}" modified between ${fromYmd} and ${toYmd}, inclusive.`);
+    lines.push(`Find SharePoint files within site "${entity}" — including its top-level document libraries and the first level of folders inside each library (depth 1) — modified between ${fromYmd} and ${toYmd}, inclusive.`);
+    lines.push('For each file, capture: filename, full server-relative path, last_modified timestamp, last_modified_by, and any external links (http/https URLs to other systems, e.g. external SharePoint sites, ADO/CRM/Jira, Loop, OneDrive, third-party docs) referenced in the document body or metadata.');
     lines.push('One CSC block per file touched in the week.');
+    lines.push('In the CSC block, place the captured external links into `topics` as a comma-separated list of bare URLs (or `_none_` if none), and use `summary` for a one-line description of what changed in the file.');
   } else {
     throw new Error(`csc-pull: unsupported source "${source}"`);
   }

package/plugin/runners/lib/layout.mjs

@@ -5,10 +5,10 @@
 import path from 'node:path';
 
 /** Sources whose record bodies are project-scoped (live under Evidence/_shared/). */
-export const SHARED_SOURCES = new Set(['crm', 'ado']);
+export const SHARED_SOURCES = new Set(['crm', 'ado', 'sharepoint']);
 
 /** Sources whose captures are per-user (live under Evidence/<alias>/<source>/). */
-export const USER_SOURCES = new Set(['email', 'teams', 'meetings', 'onenote', 'sharepoint']);
+export const USER_SOURCES = new Set(['email', 'teams', 'meetings', 'onenote']);
 
 /** Per-user note folders for shared sources (annotations only, not record bodies). */
 export const USER_NOTE_DIRS = { crm: 'crm-notes', ado: 'ado-notes' };
@@ -35,8 +35,8 @@ export function sharedRoot(project) {
 
 /** Evidence/_shared/<source>/ — only for SHARED_SOURCES. */
 export function sharedSourceDir(project, source) {
-  if (!SHARED_SOURCES.has(source) && source !== 'sharepoint') {
-    throw new Error(`layout: source "${source}" is not a shared source (allowed: ${[...SHARED_SOURCES, 'sharepoint'].join(', ')})`);
+  if (!SHARED_SOURCES.has(source)) {
+    throw new Error(`layout: source "${source}" is not a shared source (allowed: ${[...SHARED_SOURCES].join(', ')})`);
   }
   return path.join(sharedRoot(project), source);
 }

package/plugin/runners/lib/workiq.mjs

@@ -89,13 +89,16 @@ export async function ask(prompt, { bin, timeoutMs = 120_000, env = process.env,
 export function parseCscBlocks(text) {
   if (!text || typeof text !== 'string') return [];
   const blocks = [];
-  const blockquoteRe = /(^|\n)>\s*\[block:\s*([a-zA-Z0-9_.-]+)\]\s*\n((?:>\s*[^\n]*\n?)+)/g;
+  // Accept either `>` (blockquote) or `|` (table-row style) line prefixes —
+  // WorkIQ emits both depending on rendering. Trailing two-space markdown line
+  // breaks are tolerated by the per-line strip.
+  const blockquoteRe = /(^|\n)[>|]\s*\[block:\s*([a-zA-Z0-9_.-]+)\]\s*\n((?:[>|]\s*[^\n]*\n?)+)/g;
   let m;
   while ((m = blockquoteRe.exec(text)) !== null) {
     const name = m[2];
     const body = m[3]
       .split('\n')
-      .map(l => l.replace(/^>\s?/, ''))
+      .map(l => l.replace(/^[>|]\s?/, '').replace(/\s+$/, ''))
       .filter(l => l.length > 0)
       .join('\n');
     blocks.push({ name, raw: body, fields: parseKvLines(body) });

package/plugin/runners/test/integration/bootstrap.integration.test.mjs

@@ -45,7 +45,7 @@ test('scaffolds project skeleton on empty dir', async () => {
   const ledger = YAML.parse(await fs.readFile(path.join(p, 'Evidence', 'ushak', '_ledger.yml'), 'utf8'));
   assert.deepEqual(ledger, { entries: {} });
   for (const d of ['_discovery', '_deferred-retries', 'refresh-reports',
-                   'email', 'teams', 'meetings', 'onenote', 'sharepoint',
+                   'email', 'teams', 'meetings', 'onenote',
                    'crm-notes', 'ado-notes']) {
     await fs.access(path.join(p, 'Evidence', 'ushak', d));
   }