@indigoai-us/hq-cli 5.345.106 → 5.345.108

package/assets/scaffold/core/scripts/ontology-candidate.sh

@@ -1,100 +0,0 @@
-#!/usr/bin/env bash
-# ontology-candidate.sh — write one signal or entity candidate for the ontology
-# worker to garden. Deterministic, no model call. Spec:
-# core/knowledge/public/hq-core/ontology-local-spec.md
-#
-# Usage:
-#   ontology-candidate.sh write --company <co> --kind signal|entity --type <t> \
-#     --audience <company|email[,email...]> --source-ref <ref> \
-#     (--body-file <path> | --body <text>) [--created-by <email>]
-#   ontology-candidate.sh key <company|email[,email...]>   # print the audience key
-#
-# Exit 0: written, already present (no-op), or capture disabled for the company.
-# Exit 2: bad usage, unknown type, or empty body.
-set -euo pipefail
-
-root="${HQ_ROOT:-$(cd "$(dirname "$0")/../.." && pwd)}"
-here="$(cd "$(dirname "$0")" && pwd)"
-
-SIGNAL_TYPES="action_item commitment decision risk question key_point participant_contribution summary"
-ENTITY_TYPES="person project company concept"
-
-die() { echo "ontology-candidate: $*" >&2; exit 2; }
-
-sha() { if command -v sha256sum >/dev/null 2>&1; then sha256sum | awk '{print $1}'; else shasum -a 256 | awk '{print $1}'; fi; }
-
-normalize_audience() { # stdout: sorted, lowercased, trimmed, comma-joined
-  # Emails are lowercased; person/agent uids (prs_…, agt_…) keep their case.
-  printf '%s' "$1" | tr ',' '\n' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' \
-    | awk '/@/ { print tolower($0); next } { print }' \
-    | grep -v '^$' | LC_ALL=C sort -u | paste -sd, -
-}
-
-audience_key() {
-  local a; a="$(printf '%s' "$1" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
-  [ -n "$a" ] || die "empty audience"
-  if [ "$(printf '%s' "$a" | tr '[:upper:]' '[:lower:]')" = "company" ]; then echo company; return; fi
-  printf '%s' "$(normalize_audience "$a")" | sha | cut -c1-16
-}
-
-cmd="${1:-}"; shift || true
-case "$cmd" in
-  key) audience_key "${1:-}"; exit 0 ;;
-  write) ;;
-  *) die "usage: ontology-candidate.sh write|key ..." ;;
-esac
-
-co="" kind="" type="" audience="" ref="" body="" body_file="" by="${HQ_USER_EMAIL:-}"
-while [ $# -gt 0 ]; do
-  case "$1" in
-    --company) co="$2"; shift 2 ;;
-    --kind) kind="$2"; shift 2 ;;
-    --type) type="$2"; shift 2 ;;
-    --audience) audience="$2"; shift 2 ;;
-    --source-ref) ref="$2"; shift 2 ;;
-    --body) body="$2"; shift 2 ;;
-    --body-file) body_file="$2"; shift 2 ;;
-    --created-by) by="$2"; shift 2 ;;
-    *) die "unknown flag $1" ;;
-  esac
-done
-[ -n "$co" ] && [ -n "$kind" ] && [ -n "$type" ] && [ -n "$audience" ] && [ -n "$ref" ] || die "missing required flag"
-[ -n "$body_file" ] && { [ -f "$body_file" ] || die "no body file $body_file"; body="$(cat "$body_file")"; }
-body="$(printf '%s' "$body" | tr '\n' ' ' | sed 's/[[:space:]]\{1,\}/ /g;s/^ //;s/ $//')"
-[ -n "$body" ] || die "empty body"
-
-case "$kind" in
-  signal) allowed="$SIGNAL_TYPES"; flag=signals_capture; store=signals ;;
-  entity) allowed="$ENTITY_TYPES"; flag=ontology_capture; store=ontology ;;
-  *) die "kind must be signal or entity" ;;
-esac
-case " $allowed " in *" $type "*) ;; *) die "unknown $kind type '$type' (allowed: $allowed)" ;; esac
-
-enabled="$(HQ_ROOT="$root" bash "$here/knowledge-prefs.sh" get "$co" "$flag")" || exit 2
-if [ "$enabled" != "true" ]; then echo "disabled for $co ($flag=false)"; exit 0; fi
-
-key="$(audience_key "$audience")"
-norm_body="$(printf '%s' "$body" | tr '[:upper:]' '[:lower:]')"
-id="$(printf '%s\n%s\n%s\n%s' "$kind" "$type" "$key" "$norm_body" | sha)"
-
-dir="$root/companies/$co/$store/_candidates/$(date -u +%Y-%m-%d)"
-# Idempotent across days: the same candidate already written (pending or done) is a no-op.
-if [ -n "$(find "$root/companies/$co/$store/_candidates" -name "$id.md" -print -quit 2>/dev/null)" ]; then
-  echo "exists $id"; exit 0
-fi
-mkdir -p "$dir"
-{
-  echo "---"
-  echo "kind: $kind"
-  echo "type: $type"
-  echo "audience_key: $key"
-  if [ "$key" != "company" ]; then
-    echo "audience: [$(normalize_audience "$audience" | sed 's/,/, /g')]"
-  fi
-  echo "source_ref: \"$ref\""
-  [ -n "$by" ] && echo "created_by: $by"
-  echo "created_at: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
-  echo "---"
-  echo "$body"
-} > "$dir/$id.md.tmp" && mv "$dir/$id.md.tmp" "$dir/$id.md"
-echo "wrote $store/_candidates/$(basename "$dir")/$id.md"

package/assets/scaffold/core/scripts/ontology-garden.mjs

@@ -1,327 +0,0 @@
-#!/usr/bin/env node
-// ontology-garden.mjs — promote signal and entity candidates into a company's
-// local ontology and signal stores. Deterministic: no model call, no network.
-// Spec: core/knowledge/public/hq-core/ontology-local-spec.md
-//
-// Usage: node core/scripts/ontology-garden.mjs --company <co> [--hq-root <path>] [--dry-run] [--json]
-//
-// What it does, in order:
-//   1. Entity candidates  -> ontology/entities/{type}/{slug}.md (identity only).
-//      Stopword names and a name already held by an entity of another type are
-//      rejected to ontology/_rejected/{date}.jsonl.
-//   2. Signal candidates  -> signals/{type}/{sha}.md (audience company) or
-//      signals/@{key}/{type}/{sha}.md (scoped), plus signals/@{key}/_audience.yaml
-//      and signals/_index/{date}.json.
-//   3. Facts              -> for each promoted signal, every known entity whose
-//      name or alias appears in it gets a line in
-//      ontology/facts/@{key}/{type}/{slug}.md (never in the entity file).
-//      Company-audience signals bump the entity's signal_count.
-//   4. Processed candidates move to _candidates/_done/{date}/; .last-run is
-//      bumped only when at least one candidate was processed.
-// A second run over the same input changes nothing.
-import fs from "node:fs";
-import path from "node:path";
-
-const STOPWORDS = new Set([
-  "for", "calls", "call", "context", "summary", "reason", "the", "a", "an", "and",
-  "or", "this", "that", "it", "we", "they", "you", "i", "notes", "meeting", "update",
-  "question", "decision", "risk", "todo", "none", "n/a", "unknown",
-]);
-const SIGNAL_TYPES = new Set(["action_item", "commitment", "decision", "risk", "question",
-  "key_point", "participant_contribution", "summary"]);
-const ENTITY_TYPES = new Set(["person", "project", "company", "concept"]);
-
-function parseArgs(argv) {
-  const a = { dryRun: false, json: false };
-  for (let i = 0; i < argv.length; i++) {
-    const k = argv[i];
-    if (k === "--company") a.company = argv[++i];
-    else if (k === "--hq-root") a.root = argv[++i];
-    else if (k === "--dry-run") a.dryRun = true;
-    else if (k === "--json") a.json = true;
-    else { console.error(`ontology-garden: unknown flag ${k}`); process.exit(2); }
-  }
-  if (!a.company) { console.error("usage: ontology-garden.mjs --company <co> [--hq-root <p>] [--dry-run] [--json]"); process.exit(2); }
-  a.root = a.root || process.env.HQ_ROOT || path.resolve(path.dirname(new URL(import.meta.url).pathname), "../..");
-  return a;
-}
-
-// Minimal frontmatter reader for the flat key: value shape the writers emit.
-function readDoc(file) {
-  const text = fs.readFileSync(file, "utf8");
-  const m = text.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
-  if (!m) return { fm: {}, body: text.trim() };
-  const fm = {};
-  for (const line of m[1].split("\n")) {
-    const mm = line.match(/^([A-Za-z_]+):\s*(.*)$/);
-    if (!mm) continue;
-    let v = mm[2].trim();
-    if (v.startsWith("[") && v.endsWith("]")) v = v.slice(1, -1).split(",").map(s => s.trim().replace(/^["']|["']$/g, "")).filter(Boolean);
-    else v = v.replace(/^["']|["']$/g, "");
-    fm[mm[1]] = v;
-  }
-  return { fm, body: m[2].trim() };
-}
-
-function writeDoc(file, fm, body, dry) {
-  if (dry) return;
-  const lines = ["---"];
-  for (const [k, v] of Object.entries(fm)) {
-    if (v === undefined || v === null || v === "") continue;
-    lines.push(`${k}: ${Array.isArray(v) ? `[${v.join(", ")}]` : v}`);
-  }
-  lines.push("---", body, "");
-  fs.mkdirSync(path.dirname(file), { recursive: true });
-  const tmp = `${file}.tmp`;
-  fs.writeFileSync(tmp, lines.join("\n"));
-  fs.renameSync(tmp, file);
-}
-
-const slugify = s => s.toLowerCase().normalize("NFKD").replace(/[^\p{L}\p{N}]+/gu, "-").replace(/^-+|-+$/g, "").slice(0, 80);
-const today = () => new Date().toISOString().slice(0, 10);
-const now = () => new Date().toISOString().replace(/\.\d{3}Z$/, "Z");
-const yyyyMm = () => new Date().toISOString().slice(0, 7);
-
-// ── Fact-file compaction ─────────────────────────────────────────────
-// Fact pages append one line per signal-mention of an entity (see step 3
-// below). Without a bound a hot entity accumulates every mention it has
-// ever received, so the page becomes a flat log. Compaction keeps the
-// most recent N lines inline and rolls the older ones into a dated,
-// append-only archive next to the fact file. `signal_count` on the
-// entity itself is unaffected — it is the true total.
-const BACKREF_COMPACT_THRESHOLD = Number(process.env.HQ_ONTOLOGY_BACKREF_THRESHOLD || 50);
-const BACKREF_KEEP_INLINE = Number(process.env.HQ_ONTOLOGY_BACKREF_KEEP_INLINE || 25);
-const SUMMARY_MARKER = "<!-- backref-archive -->";
-
-function isBackrefLine(line) { return /^- \[[a-z_]+\] .+ \(signal [0-9a-f]{12}\)\s*$/.test(line); }
-function isSummaryLine(line) { return line.includes(SUMMARY_MARKER); }
-
-function readArchiveIndex(file) {
-  if (!fs.existsSync(file)) return { version: 1, signalIds: [] };
-  try {
-    const parsed = JSON.parse(fs.readFileSync(file, "utf8"));
-    if (parsed && Array.isArray(parsed.signalIds)) {
-      return { version: 1, signalIds: parsed.signalIds.filter(s => typeof s === "string") };
-    }
-  } catch (err) {
-    console.warn(`ontology-garden: archive index unreadable at ${file}: ${err.message || err}; skipping compaction`);
-  }
-  // Present but unreadable: never compact on top of it, or the rewrite would
-  // drop every archived id it lists.
-  return { version: 1, signalIds: [], unreadable: true };
-}
-
-// Compact `body` when its backref-line count exceeds the threshold.
-// Returns { body, archivedLines, archivedIds, changed }.
-function compactFactBody(body, { slug, archivedTotalBefore, dry }) {
-  const lines = body.split("\n").map(l => l.trimEnd()).filter(l => l.length > 0);
-  const backrefLines = lines.filter(isBackrefLine);
-  const other = lines.filter(l => !isBackrefLine(l) && !isSummaryLine(l));
-  if (backrefLines.length <= BACKREF_COMPACT_THRESHOLD) {
-    return { body, archivedLines: [], archivedIds: [], changed: false };
-  }
-  const archiveCount = backrefLines.length - BACKREF_KEEP_INLINE;
-  const archivedLines = backrefLines.slice(0, archiveCount);
-  const inlineLines = backrefLines.slice(archiveCount);
-  const archivedIds = archivedLines
-    .map(l => (l.match(/\(signal ([0-9a-f]{12})\)$/) || [])[1])
-    .filter(Boolean);
-  const archivedTotal = archivedTotalBefore + archivedLines.length;
-  const stamp = yyyyMm();
-  const summary = `- ${SUMMARY_MARKER} _${archivedTotal} older mention${archivedTotal === 1 ? "" : "s"} archived in [history](./${slug}.history/${stamp}.md)_`;
-  const nextBody = [summary, ...other, ...inlineLines].join("\n");
-  return { body: nextBody, archivedLines, archivedIds, changed: true, stamp };
-}
-
-function appendArchiveFile(file, isoDate, lines, dry) {
-  if (dry) return;
-  const batch = `\n<!-- archived ${isoDate} -->\n${lines.join("\n")}\n`;
-  const prev = fs.existsSync(file) ? fs.readFileSync(file, "utf8").replace(/\s+$/, "") : "";
-  const next = prev ? `${prev}\n${batch}` : `# Back-reference archive\n${batch}`;
-  fs.mkdirSync(path.dirname(file), { recursive: true });
-  const tmp = `${file}.tmp`;
-  fs.writeFileSync(tmp, next);
-  fs.renameSync(tmp, file);
-}
-
-function listCandidates(dir) {
-  if (!fs.existsSync(dir)) return [];
-  const out = [];
-  for (const d of fs.readdirSync(dir).sort()) {
-    if (d === "_done") continue;
-    const full = path.join(dir, d);
-    if (!fs.statSync(full).isDirectory()) continue;
-    for (const f of fs.readdirSync(full).sort()) if (f.endsWith(".md")) out.push(path.join(full, f));
-  }
-  return out;
-}
-
-function main() {
-  const a = parseArgs(process.argv.slice(2));
-  const backrefThreshold = Number(process.env.HQ_ONTOLOGY_BACKREF_THRESHOLD || 50);
-  const backrefKeepInline = Number(process.env.HQ_ONTOLOGY_BACKREF_KEEP_INLINE || 25);
-  if (!(backrefThreshold > backrefKeepInline)) {
-    console.error(`ontology-garden: BACKREF_THRESHOLD (${backrefThreshold}) must be greater than BACKREF_KEEP_INLINE (${backrefKeepInline})`);
-    process.exit(1);
-  }
-  const co = path.join(a.root, "companies", a.company);
-  if (!fs.existsSync(co)) { console.error(`ontology-garden: no company ${a.company}`); process.exit(2); }
-  const ont = path.join(co, "ontology"), sig = path.join(co, "signals");
-  const report = { company: a.company, entities_created: 0, entities_updated: 0, signals_promoted: 0,
-    facts_written: 0, rejected: 0, candidates_processed: 0, dry_run: a.dryRun };
-  const rejected = [];
-  const moves = [];
-
-  // Load the current entity index: slug -> {type, file, fm, names[]}
-  const entities = new Map();
-  const nameIndex = new Map(); // lowercased name/alias -> slug
-  const entDir = path.join(ont, "entities");
-  if (fs.existsSync(entDir)) for (const t of fs.readdirSync(entDir)) {
-    const td = path.join(entDir, t);
-    if (!fs.statSync(td).isDirectory()) continue;
-    for (const f of fs.readdirSync(td)) {
-      if (!f.endsWith(".md")) continue;
-      const file = path.join(td, f); const { fm } = readDoc(file);
-      const slug = fm.slug || f.replace(/\.md$/, "");
-      const names = [fm.canonical_name, ...(Array.isArray(fm.aliases) ? fm.aliases : [])].filter(Boolean);
-      entities.set(slug, { type: fm.type || t, file, fm, names });
-      for (const n of names) nameIndex.set(n.toLowerCase(), slug);
-    }
-  }
-
-  // 1. Entity candidates
-  for (const file of listCandidates(path.join(ont, "_candidates"))) {
-    const { fm, body } = readDoc(file);
-    const name = body.split("\n")[0].trim();
-    const type = fm.type;
-    const reject = reason => { rejected.push({ at: now(), name, type, reason, candidate: path.relative(co, file) }); report.rejected++; };
-    if (!ENTITY_TYPES.has(type)) reject("unknown-type");
-    else if (!name || STOPWORDS.has(name.toLowerCase()) || name.length < 2) reject("stopword");
-    else {
-      const slug = slugify(name);
-      const held = nameIndex.get(name.toLowerCase());
-      const existing = entities.get(held || slug);
-      if (existing && existing.type !== type) reject(`type-conflict:${existing.type}`);
-      else if (existing) {
-        const aliases = new Set(Array.isArray(existing.fm.aliases) ? existing.fm.aliases : []);
-        if (name !== existing.fm.canonical_name && !aliases.has(name)) {
-          aliases.add(name); existing.fm.aliases = [...aliases]; existing.fm.last_updated = now();
-          writeDoc(existing.file, existing.fm, "", a.dryRun); report.entities_updated++;
-          nameIndex.set(name.toLowerCase(), held || slug);
-        }
-      } else {
-        const efile = path.join(ont, "entities", type, `${slug}.md`);
-        const efm = { type, canonical_name: name, slug, aliases: [], signal_count: 0,
-          first_seen: fm.created_at || now(), last_updated: now(), enriched_by: "ontology-worker" };
-        writeDoc(efile, efm, "", a.dryRun);
-        entities.set(slug, { type, file: efile, fm: efm, names: [name] });
-        nameIndex.set(name.toLowerCase(), slug);
-        report.entities_created++;
-      }
-    }
-    moves.push(file); report.candidates_processed++;
-  }
-
-  // 2 + 3. Signal candidates -> signals + facts
-  const index = [];
-  for (const file of listCandidates(path.join(sig, "_candidates"))) {
-    const { fm, body } = readDoc(file);
-    const type = fm.type; const key = fm.audience_key;
-    moves.push(file); report.candidates_processed++;
-    if (!SIGNAL_TYPES.has(type) || !key || !body) {
-      rejected.push({ at: now(), type, reason: !key ? "missing-audience" : "invalid", candidate: path.relative(co, file) }); report.rejected++;
-      continue;
-    }
-    if (key !== "company" && !(Array.isArray(fm.audience) && fm.audience.length)) {
-      rejected.push({ at: now(), type, reason: "scoped-without-principals", candidate: path.relative(co, file) }); report.rejected++;
-      continue;
-    }
-    const id = path.basename(file, ".md");
-    const scopeDir = key === "company" ? sig : path.join(sig, `@${key}`);
-    const out = path.join(scopeDir, type, `${id}.md`);
-    if (!fs.existsSync(out)) {
-      writeDoc(out, { signal_id: id, type, canonical_content: JSON.stringify(body.split("\n")[0]), audience_key: key,
-        source_ref: JSON.stringify(fm.source_ref || ""), created_at: fm.created_at || now() }, body, a.dryRun);
-      report.signals_promoted++;
-      index.push({ signal_id: id, type, audience_key: key, path: path.relative(co, out) });
-      if (key !== "company") {
-        const audFile = path.join(scopeDir, "_audience.yaml");
-        if (!fs.existsSync(audFile) && !a.dryRun) {
-          fs.mkdirSync(scopeDir, { recursive: true });
-          fs.writeFileSync(audFile, `audience_key: ${key}\nprincipals: [${fm.audience.join(", ")}]\n`);
-        }
-      }
-      // Facts: every known entity named in the signal.
-      const lower = body.toLowerCase();
-      const hit = new Set();
-      for (const [n, slug] of nameIndex) {
-        if (n.length < 3) continue;
-        const re = new RegExp(`(^|[^\\p{L}\\p{N}])${n.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}($|[^\\p{L}\\p{N}])`, "u");
-        if (re.test(lower)) hit.add(slug);
-      }
-      for (const slug of hit) {
-        const e = entities.get(slug);
-        const ffile = path.join(ont, "facts", `@${key}`, e.type, `${slug}.md`);
-        const sid12 = id.slice(0, 12);
-        const line = `- [${type}] ${body.split("\n")[0]} (signal ${sid12})`;
-        const prev = fs.existsSync(ffile) ? readDoc(ffile) : { fm: { entity: slug, type: e.type, audience_key: key }, body: "" };
-        const idxFile = path.join(path.dirname(ffile), `${slug}.history`, "_index.json");
-        const archiveIndex = readArchiveIndex(idxFile);
-        const alreadyArchived = archiveIndex.signalIds.includes(sid12);
-        if (!prev.body.includes(`signal ${sid12}`) && !alreadyArchived) {
-          let nextBody = [prev.body, line].filter(Boolean).join("\n");
-          const compact = compactFactBody(nextBody, {
-            slug,
-            archivedTotalBefore: archiveIndex.signalIds.length,
-            dry: a.dryRun,
-          });
-          if (compact.changed && !archiveIndex.unreadable) {
-            const archiveFile = path.join(path.dirname(ffile), `${slug}.history`, `${compact.stamp}.md`);
-            appendArchiveFile(archiveFile, today(), compact.archivedLines, a.dryRun);
-            if (!a.dryRun) {
-              const nextIdx = { version: 1, signalIds: [...archiveIndex.signalIds, ...compact.archivedIds] };
-              fs.mkdirSync(path.dirname(idxFile), { recursive: true });
-              fs.writeFileSync(idxFile, JSON.stringify(nextIdx) + "\n");
-            }
-            nextBody = compact.body;
-          }
-          writeDoc(ffile, prev.fm, nextBody, a.dryRun);
-          report.facts_written++;
-        }
-        if (key === "company") {
-          e.fm.signal_count = String((parseInt(e.fm.signal_count || "0", 10) || 0) + 1);
-          e.fm.last_updated = now();
-          writeDoc(e.file, e.fm, "", a.dryRun);
-        }
-      }
-    }
-  }
-
-  if (!a.dryRun) {
-    if (index.length) {
-      const f = path.join(sig, "_index", `${today()}.json`);
-      const prev = fs.existsSync(f) ? JSON.parse(fs.readFileSync(f, "utf8")) : [];
-      fs.mkdirSync(path.dirname(f), { recursive: true });
-      fs.writeFileSync(f, JSON.stringify([...prev, ...index], null, 2) + "\n");
-    }
-    if (rejected.length) {
-      const f = path.join(ont, "_rejected", `${today()}.jsonl`);
-      fs.mkdirSync(path.dirname(f), { recursive: true });
-      fs.appendFileSync(f, rejected.map(r => JSON.stringify(r)).join("\n") + "\n");
-    }
-    for (const file of moves) {
-      const rel = path.relative(path.dirname(path.dirname(file)), file); // {date}/{sha}.md
-      const dest = path.join(path.dirname(path.dirname(file)), "_done", rel);
-      fs.mkdirSync(path.dirname(dest), { recursive: true });
-      fs.renameSync(file, dest);
-    }
-    if (report.candidates_processed > 0) {
-      fs.mkdirSync(ont, { recursive: true });
-      fs.writeFileSync(path.join(ont, ".last-run"), `${Date.now()}\n`);
-    }
-  }
-  if (a.json) console.log(JSON.stringify(report));
-  else console.log(`garden ${a.company}: +${report.entities_created} entities, ~${report.entities_updated} updated, +${report.signals_promoted} signals, +${report.facts_written} facts, ${report.rejected} rejected, ${report.candidates_processed} candidates${a.dryRun ? " (dry run)" : ""}`);
-}
-
-main();

package/assets/scaffold/core/scripts/ontology-source-worklist.sh

@@ -1,226 +0,0 @@
-#!/usr/bin/env bash
-# ontology-source-worklist.sh — list a declared source's unprocessed items with
-# each item's resolved audience. The ontology worker's process-source skill
-# reads this list, extracts candidates from each item, and marks it done.
-# Specs: source-yaml-spec.md, ontology-local-spec.md (core/knowledge/public/hq-core/)
-#
-# Usage:
-#   ontology-source-worklist.sh --company <co> --channel <ch> [--limit N] [--force]
-#   ontology-source-worklist.sh --company <co> --channel <ch> --mark-done <item-file>
-#
-# Output: one JSON object per line:
-#   {"file":"sources/meetings/x.md","audience_key":"…","audience":["prs_…","a@x.com"]}
-#   {"file":"sources/meetings/y.md","skip":"no-audience"}
-# Audience by audience_rule:
-#   attendees / thread / channel — the item's `audience:` or `attendees:` list if
-#     it has one, else the direct read grants on the item's vault key
-#     (hq files acl), which is how cloud ingestion records who was privy.
-#   explicit — the item's own `audience:` frontmatter only.
-#   company  — "company".
-# An item with no resolvable audience is emitted as a skip, never as company.
-# Exit 0; 2 bad usage / invalid source.yaml; 4 run: cloud source without --force.
-set -uo pipefail
-HQ="${HQ_BIN:-hq}"
-root="${HQ_ROOT:-$(cd "$(dirname "$0")/../.." && pwd)}"
-here="$(cd "$(dirname "$0")" && pwd)"
-co="" ch="" limit=50 force=0 done_item=""
-while [ $# -gt 0 ]; do
-  case "$1" in
-    --company) co="$2"; shift 2 ;; --channel) ch="$2"; shift 2 ;; --limit) limit="$2"; shift 2 ;;
-    --force) force=1; shift ;; --mark-done) done_item="$2"; shift 2 ;;
-    *) echo "unknown flag $1" >&2; exit 2 ;;
-  esac
-done
-[ -n "$co" ] && [ -n "$ch" ] || { echo "usage: ontology-source-worklist.sh --company <co> --channel <ch> [--limit N] [--force] [--mark-done <file>]" >&2; exit 2; }
-D="$root/companies/$co/sources/$ch"; Y="$D/source.yaml"; ledger="$D/.processed"; skip_ledger="$D/.skipped"
-# Bumped when the audience parser gains new recognisable frontmatter fields; items
-# previously skipped as no-audience under an older PARSER_VERSION get re-evaluated
-# on the next run instead of staying stuck forever (see feedback_b8d61e28).
-PARSER_VERSION=2
-bash "$here/source-yaml-validate.sh" "$Y" >/dev/null || { bash "$here/source-yaml-validate.sh" "$Y" >&2; exit 2; }
-
-if [ -n "$done_item" ]; then printf '%s\t%s\n' "$(basename "$done_item")" "$PARSER_VERSION" >> "$ledger"; exit 0; fi
-
-# Migrate `.processed` to parser-version stamps so a successfully processed item
-# stays processed across runs. Entries are `basename\tversion`; new mark-done
-# writes always include the stamp. Convergence rule for legacy UNSTAMPED
-# entries from the pre-fix slack-dm bug (254 DMs recorded in `.processed` as
-# plain basenames while no candidates were ever written): drop ones whose file
-# carries `resolved_participants:` (the exact bug signature) so the main loop
-# re-evaluates them exactly once; the next mark-done writes them back stamped
-# at the current version. Any other unstamped entry is upgraded in place to a
-# stamp — never re-stripped again. Stamped entries at a version below current
-# PARSER_VERSION are also dropped so a parser upgrade re-evaluates them.
-# ontology-candidate ids are content-addressed, so a true reprocess is idempotent.
-if [ -f "$ledger" ]; then
-  tmp="$ledger.heal.$$"
-  : > "$tmp"
-  changed=0
-  while IFS=$'\t' read -r b ver || [ -n "$b" ]; do
-    b="${b%$'\r'}"; ver="${ver%$'\r'}"
-    [ -n "$b" ] || continue
-    if [ -n "${ver:-}" ]; then
-      if [ "$ver" -lt "$PARSER_VERSION" ] 2>/dev/null; then changed=1; continue; fi
-      printf '%s\t%s\n' "$b" "$ver" >> "$tmp"
-      continue
-    fi
-    item="$D/$b"
-    if [ -f "$item" ] && awk '/^---$/{fm++; if(fm==2) exit} fm==1 && /^resolved_participants:/{found=1; exit} END{exit !found}' "$item" 2>/dev/null; then
-      changed=1; continue
-    fi
-    printf '%s\t%s\n' "$b" "$PARSER_VERSION" >> "$tmp"
-    changed=1
-  done < "$ledger"
-  if [ "$changed" = 1 ]; then mv "$tmp" "$ledger"; else rm -f "$tmp"; fi
-fi
-is_processed() { # 0 if $1 is recorded in .processed at >= current parser version (or unstamped — treated as current after migration)
-  [ -f "$ledger" ] || return 1
-  awk -F '\t' -v b="$1" -v v="$PARSER_VERSION" '
-    { sub(/\r$/, "") }
-    $1==b { if (NF < 2 || $2=="") { found=1; exit } if (($2+0) >= v) { found=1; exit } }
-    END { exit !found }
-  ' "$ledger"
-}
-
-# Walk `.skipped`: entries whose recorded version is < current PARSER_VERSION
-# are dropped so the main loop re-evaluates them. Entries at the current
-# version stay as the ignore set for this run.
-declare_skipped=""
-if [ -f "$skip_ledger" ]; then
-  tmp="$skip_ledger.rev.$$"
-  : > "$tmp"
-  changed=0
-  while IFS=$'\t' read -r b reason ver || [ -n "$b" ]; do
-    [ -n "$b" ] || continue
-    if [ -z "${ver:-}" ] || [ "$ver" -lt "$PARSER_VERSION" ] 2>/dev/null; then
-      changed=1; continue
-    fi
-    printf '%s\t%s\t%s\n' "$b" "$reason" "$ver" >> "$tmp"
-  done < "$skip_ledger"
-  if [ "$changed" = 1 ]; then mv "$tmp" "$skip_ledger"; else rm -f "$tmp"; fi
-fi
-skipped_at_current() { # 0 if $1 is recorded as skipped at the current parser version
-  [ -f "$skip_ledger" ] || return 1
-  awk -F '\t' -v b="$1" -v v="$PARSER_VERSION" '$1==b && $3==v { found=1; exit } END { exit !found }' "$skip_ledger"
-}
-
-get() { awk -v k="$1" '$0 ~ "^"k":" { sub("^"k":[[:space:]]*", ""); gsub(/^["\x27]|["\x27]$/, ""); print; exit }' "$Y"; }
-run="$(get run)"; rule="$(get audience_rule)"
-if [ "$run" = cloud ] && [ "$force" = 0 ]; then echo "assigned to cloud agent (run: cloud); pass --force to run locally" >&2; exit 4; fi
-
-item_list() { # frontmatter list field: inline [a, b] or block "- a"
-  awk -v k="$2" '
-    BEGIN{fm=0}
-    /^---$/ { fm++; if (fm==2) exit; next }
-    fm==1 && $0 ~ "^"k":" { v=$0; sub("^"k":[[:space:]]*", "", v)
-      if (v ~ /^\[/) { gsub(/[\[\]"\x27]/, "", v); n=split(v, a, ","); for(i=1;i<=n;i++){gsub(/^[ \t]+|[ \t]+$/, "", a[i]); if(a[i]!="") print a[i]}; exit }
-      inlist=1; next }
-    fm==1 && inlist && /^[[:space:]]*-[[:space:]]/ { v=$0; sub(/^[[:space:]]*-[[:space:]]*/, "", v); gsub(/["\x27]/, "", v); print v; next }
-    fm==1 && inlist { exit }
-  ' "$1"
-}
-
-# participant_ids: for a frontmatter block list of mappings under key $2 (e.g.
-# resolved_participants:), emit one id per entry: the entry's `person_uid:` if
-# present, otherwise its `email:`. Entries with only `slack_user_id:` (no
-# person_uid, no email) are skipped — the audience must stay exactly the
-# resolvable DM participants, never widened. Scope: ONLY the mapping list
-# directly under $2 in frontmatter.
-participant_ids() {
-  awk -v k="$2" '
-    { sub(/\r$/, "") }
-    BEGIN{fm=0; inlist=0; have_pid=0; have_email=0; pid=""; em=""}
-    function flush() {
-      if (have_pid && pid != "") { print pid }
-      else if (have_email && em != "") { print em }
-      have_pid=0; have_email=0; pid=""; em=""
-    }
-    /^---$/ { fm++; if (fm==2) { if (inlist) flush(); exit } next }
-    fm==1 && $0 ~ "^"k":" { inlist=1; next }
-    fm==1 && inlist {
-      # end of list = a line that is not indented beyond col 0 and not a bare "-"
-      if ($0 ~ /^[^ \t-]/) { flush(); inlist=0; next }
-      is_dash = ($0 ~ /^[[:space:]]*-[[:space:]]/)
-      if (is_dash) {
-        flush()
-        s=$0; sub(/^[[:space:]]*-[[:space:]]*/, "", s)
-        # inline mapping (JSON-style) on same line as the dash
-        if (s ~ /^\{/) {
-          gsub(/[{}"\x27]/, "", s); n=split(s, kv, ",")
-          for (i=1;i<=n;i++) {
-            gsub(/^[ \t]+|[ \t]+$/, "", kv[i])
-            if (match(kv[i], /^person_uid[[:space:]]*:[[:space:]]*/)) { pid=substr(kv[i], RLENGTH+1); have_pid=1 }
-            else if (match(kv[i], /^email[[:space:]]*:[[:space:]]*/)) { em=substr(kv[i], RLENGTH+1); have_email=1 }
-          }
-          flush()
-          next
-        }
-        # First mapping key on same line as the dash, e.g.
-        #   - person_uid: prs_01ALICE
-        # Fall through to the key/value handlers below with the de-dashed line.
-        line = s
-      } else {
-        line = $0; sub(/^[[:space:]]+/, "", line)
-      }
-      if (line ~ /^person_uid:/) {
-        v=line; sub(/^person_uid:[[:space:]]*/, "", v); gsub(/["\x27]/, "", v); gsub(/^[ \t]+|[ \t]+$/, "", v)
-        if (v != "") { pid=v; have_pid=1 }
-        next
-      }
-      if (line ~ /^email:/) {
-        v=line; sub(/^email:[[:space:]]*/, "", v); gsub(/["\x27]/, "", v); gsub(/^[ \t]+|[ \t]+$/, "", v)
-        if (v != "") { em=v; have_email=1 }
-        next
-      }
-    }
-    END { if (inlist) flush() }
-  ' "$1"
-}
-
-n=0
-for f in "$D"/*.md; do
-  [ -f "$f" ] || continue
-  b="$(basename "$f")"
-  is_processed "$b" && continue
-  skipped_at_current "$b" && continue
-  [ "$n" -ge "$limit" ] && break
-  n=$((n+1))
-  rel="sources/$ch/$b"
-  aud=""
-  case "$rule" in
-    company) aud="company" ;;
-    explicit) aud="$(item_list "$f" audience | paste -sd, -)" ;;
-    *)
-      aud="$(item_list "$f" audience | paste -sd, -)"
-      [ -n "$aud" ] || aud="$(item_list "$f" attendees | grep -E '@|^(prs|agt)_' | paste -sd, -)"
-      # Slack DM items (hq-pro inbound slack-personal-dm.ts) only carry
-      # resolved_participants: a block list of mappings with person_uid /
-      # slack_user_id / email. Derive audience from those, scoped exactly to
-      # the DM's own participants (never widened to company). Only entries
-      # whose person_uid or email resolves become audience ids; a slack_user_id
-      # alone is not a usable audience key, so that entry is dropped — matching
-      # the frontmatter contract exercised in hq-pro
-      # test/sources/inbound/slack-personal-dm.test.ts.
-      [ -n "$aud" ] || aud="$(participant_ids "$f" resolved_participants | grep -E '@|^(prs|agt)_' | paste -sd, -)"
-      if [ -z "$aud" ]; then
-        # A transient ACL or jq failure must not turn a source into a durable
-        # no-audience skip. Retry once, while treating valid empty ACL output as
-        # a completed lookup that needs no retry.
-        for attempt in 1 2; do
-          acl_output=""
-          if acl_output="$("$HQ" files --company "$co" acl "$rel" --json 2>/dev/null \
-            | jq -r '[.direct[]? | select(.granteeType=="person" or .granteeType=="email" or .granteeType=="agent") | select(.permission=="read" or .permission=="write" or .permission=="admin") | .granteeId] | unique | join(",")' 2>/dev/null)"; then
-            aud="$acl_output"
-            break
-          fi
-          aud=""
-        done
-      fi ;;
-  esac
-  if [ -z "$aud" ]; then
-    printf '%s\t%s\t%s\n' "$b" "no-audience" "$PARSER_VERSION" >> "$skip_ledger"
-    jq -cn --arg f "$rel" '{file:$f, skip:"no-audience"}'; continue
-  fi
-  key="$(HQ_ROOT="$root" bash "$here/ontology-candidate.sh" key "$aud")"
-  jq -cn --arg f "$rel" --arg k "$key" --arg a "$aud" '{file:$f, audience_key:$k, audience:($a|split(","))}'
-done