@polycode-projects/the-mechanical-code-talker 2.5.2 → 2.5.4

package/src/domain/dialogue-acts.mjs

@@ -0,0 +1,160 @@
+// The closed dialogue-act vocabulary: tmct's turn types named to ISO 24617-2's
+// communicative functions, using the standard's own camelCase data-category
+// spellings. The router already sorts every turn into a lane, so the label a
+// consumer attaches is a fixed lookup over a decision already made — a table,
+// never a classifier, and no model anywhere.
+//
+// The honesty-preserving line runs through this table: tmct's miss is
+// autoNegative in the autoFeedback dimension — the sender reporting that its
+// OWN processing could not ground the turn — never a task answer. A flat
+// intent set collapses that distinction; keeping it is why the standard is
+// worth adopting over an ad-hoc set.
+//
+// The subset here covers what tmct's lanes produce. The standard's responsive
+// accept/decline family is deliberately absent; a session extending the table
+// should know the standard's own cross-wiring: accepting a request or a
+// suggestion is a COMMISSIVE, while accepting an offer is a DIRECTIVE, and the
+// address* functions are the general "dealing with" nodes those specialise.
+//
+// Source of the names: docs/references/schemas/iso-24617-2-dialogue-acts.md —
+// a good-faith reading of the editors' LREC 2020 paper and the public DIS
+// draft, not a quotation of the paywalled published text.
+
+/** The standard's ten dimensions (second edition), camelCase per its Annex F
+ *  data-category names. Every act in DIALOGUE_ACTS carries exactly one. */
+export const DIALOGUE_ACT_DIMENSIONS = Object.freeze([
+  "task",
+  "autoFeedback",
+  "alloFeedback",
+  "turnManagement",
+  "timeManagement",
+  "discourseStructuring",
+  "ownCommunicationManagement",
+  "partnerCommunicationManagement",
+  "socialObligationsManagement",
+  "contactManagement",
+]);
+
+/** Every communicative function tmct's lanes cover. General-purpose functions
+ *  (questions, informs, directives, commissives) carry the dimension tmct
+ *  annotates them in — task; the feedback and social functions carry the fixed
+ *  dimension the standard gives them. `gloss` says what the function is on the
+ *  standard's terms, with tmct's use in brackets where one exists. */
+export const DIALOGUE_ACTS = Object.freeze({
+  propositionalQuestion: Object.freeze({
+    dimension: "task",
+    gloss: "a yes/no question about whether a proposition holds ('does X import Y?')",
+  }),
+  checkQuestion: Object.freeze({
+    dimension: "task",
+    gloss: "a yes/no question whose asker already expects the answer yes ('..., right?')",
+  }),
+  setQuestion: Object.freeze({
+    dimension: "task",
+    gloss: "a wh-question asking for the members of a set ('what does X import?')",
+  }),
+  choiceQuestion: Object.freeze({
+    dimension: "task",
+    gloss: "a question asking which of the listed alternatives holds ('is X a module or a class?')",
+  }),
+  inform: Object.freeze({
+    dimension: "task",
+    gloss: "a declarative telling the addressee something (a teach turn; also tmct explaining its own function)",
+  }),
+  answer: Object.freeze({
+    dimension: "task",
+    gloss: "an inform that discharges a question just asked (an answer grounded in the graph)",
+  }),
+  confirm: Object.freeze({
+    dimension: "task",
+    gloss: "an answer 'yes' to a check question",
+  }),
+  disconfirm: Object.freeze({
+    dimension: "task",
+    gloss: "an answer 'no' to a check question",
+  }),
+  agreement: Object.freeze({
+    dimension: "task",
+    gloss: "an inform stating that the speaker holds what the addressee just stated to be true",
+  }),
+  disagreement: Object.freeze({
+    dimension: "task",
+    gloss: "an inform stating that the speaker holds what the addressee just stated to be false",
+  }),
+  correction: Object.freeze({
+    dimension: "task",
+    gloss: "a disagreement that also supplies the replacement ('no, I meant Y')",
+  }),
+  request: Object.freeze({
+    dimension: "task",
+    gloss: "asks the addressee to perform an action ('solve it' — a goal turn)",
+  }),
+  instruct: Object.freeze({
+    dimension: "task",
+    gloss: "a request the addressee is expected to carry out without negotiation (a bare imperative)",
+  }),
+  suggestion: Object.freeze({
+    dimension: "task",
+    gloss: "puts an action forward as advisable without claiming authority over the addressee",
+  }),
+  offer: Object.freeze({
+    dimension: "task",
+    gloss: "commits the speaker to an action, conditional on the addressee wanting it",
+  }),
+  autoPositive: Object.freeze({
+    dimension: "autoFeedback",
+    gloss: "the sender reports its own processing of the previous turn succeeded (an acknowledgement)",
+  }),
+  autoNegative: Object.freeze({
+    dimension: "autoFeedback",
+    gloss: "the sender reports its own processing of the previous turn failed — tmct's honest miss",
+  }),
+  initialGreeting: Object.freeze({
+    dimension: "socialObligationsManagement",
+    gloss: "opens an exchange of greetings ('hi')",
+  }),
+  returnGreeting: Object.freeze({
+    dimension: "socialObligationsManagement",
+    gloss: "answers a greeting with a greeting",
+  }),
+  thanking: Object.freeze({
+    dimension: "socialObligationsManagement",
+    gloss: "expresses gratitude for something the addressee did",
+  }),
+  apology: Object.freeze({
+    dimension: "socialObligationsManagement",
+    gloss: "expresses regret for something the speaker did",
+  }),
+});
+
+/** The router-lane → dialogue-act lookup the chat surface attaches per turn: a
+ *  deterministic map over the lane the router already picked. Keys name router
+ *  lanes; values name DIALOGUE_ACTS entries. The honest-miss row is the one
+ *  that must never drift: a miss is autoFeedback about tmct's own processing,
+ *  not a task answer. */
+export const LANE_DIALOGUE_ACTS = Object.freeze({
+  teach: "inform",
+  "ask-set": "setQuestion",
+  "ask-propositional": "propositionalQuestion",
+  goal: "request",
+  imperative: "instruct",
+  "honest-miss": "autoNegative",
+  greeting: "initialGreeting",
+  thanks: "thanking",
+  help: "inform",
+  // The guessing game's turns are task-dimension: a reply that discharges
+  // the other side's move (a hint, a win, a rebuttal) is an answer; tmct
+  // stating its own move (an opening, its next guess) is an inform.
+  "game-answer": "answer",
+  "game-inform": "inform",
+});
+
+/** The dialogue act a router lane resolves to, with its dimension — or null
+ *  for a lane the lookup does not cover (the caller keeps its own naming). */
+export function dialogueActForLane(lane) {
+  // hasOwn, so an inherited name ("constructor") is uncovered, never a hit
+  // found on the prototype chain.
+  if (typeof lane !== "string" || !Object.hasOwn(LANE_DIALOGUE_ACTS, lane)) return null;
+  const act = LANE_DIALOGUE_ACTS[lane];
+  return { act, dimension: DIALOGUE_ACTS[act].dimension };
+}

package/src/domain/interpret/normalize.mjs

@@ -153,14 +153,14 @@ const TELL_ME_WRAPPER_RE = /^tell\s+me\s+(.+?)\??$/i;
 const KNOW_WRAPPER_RE = /^do\s+you\s+know\s+(.+?)\??$/i;
 /** "i'd like to know <Q>" / "i want to know <Q>" -> "<Q>", same
  *  interrogative-remainder gate as KNOW_WRAPPER_RE. */
-const WANT_KNOW_WRAPPER_RE = /^i(?:'d|\s+would)?\s+(?:like|want|need)\s+to\s+know\s+(.+?)\??$/i;
+const WANT_KNOW_WRAPPER_RE = /^i(?:'d|\s+would)?\s+(?:(?:like|want|need)\s+to|wanna)\s+know\s+(.+?)\??$/i;
 /** "i was wondering <Q>" / "i wondered <Q>" / "i'm curious <Q>" -> "<Q>", the
  *  same wrapper family and the same interrogative-remainder gate. The modal
  *  form ("could you tell me what a dog is") was already unwrapped; this one
  *  states the wish rather than asking, which is how it fell through every
  *  frame. The optional "if"/"whether"/"about" tail carries the same clause
  *  MODAL_WRAPPER_RE's own remainder does. */
-const WONDERING_WRAPPER_RE = /^i(?:\s+was|\s+am|'m)?\s+(?:just\s+)?(?:wonder(?:ing|ed)|curious)(?:\s+(?:if|whether|about))?\s+(.+?)\??$/i;
+const WONDERING_WRAPPER_RE = /^i(?:\s+was|\s+am|'m)?\s+(?:just\s+)?(?:wonder(?:ing|ed)?|curious)(?:\s+(?:if|whether|about))?\s+(.+?)\??$/i;
 /** EMBEDDED-QUESTION DE-INVERSION: the wrappers above unwrap "could you
  *  tell me what a dog is" down to the embedded clause "what a dog is",
  *  which keeps declarative word order — nothing downstream parses it. Fold
@@ -498,6 +498,18 @@ const PHRASING_FRAMES = Object.freeze([
   // NEEDS-TESTS → the untested-module survey.
   { re: /^what\s+needs\s+(?:to\s+be\s+)?(?:a\s+)?(?:tested|tests?|testing|coverage|covering)\??$/i, to: () => "untested modules" },
 
+  // NO-TESTS → the same untested survey, attributive form. "no tests" would
+  // otherwise read as a literal object term ("…defines no tests") and the
+  // resulting empty answers as "it has none" to a coverage question.
+  {
+    re: /^(?:which|what|list|show(?:\s+me)?|find)?\s*(?:the\s+|all\s+)?([a-z][a-z-]*?)\s+(?:(?:that|which)\s+)?(?:have|has|having|with)\s+no\s+tests?\??$/i,
+    to: (m) => `untested ${m[1].toLowerCase()}`,
+  },
+  {
+    re: /^(?:which|what|list|show(?:\s+me)?|find)?\s*(?:the\s+|all\s+)?([a-z][a-z-]*?)\s+without\s+(?:any\s+)?tests?\??$/i,
+    to: (m) => `untested ${m[1].toLowerCase()}`,
+  },
+
   // DOES-X-VERB-ANYTHING-ELSE → "what does X <verb>" (drops the placeholder
   // "anything/something else" object, which otherwise made the two parse
   // strategies disagree on the span). Anchored to VERB_TO_KIND so it can't

package/src/domain/interpret/strategies/keywords.mjs

@@ -222,7 +222,16 @@ export function parseKeywordSpot(text, nlp = null) {
     if (INHERITS_REVERSE_VERBS.includes(verbPhrase)) [subject, object] = [object, subject];
     return stamp({ shape: "ask", entityType: null, modifier: "direct", kind, subject, object });
   }
-  if (afterText) return stamp({ shape: "reverse", entityType, modifier, kind, object: afterText });
+  if (afterText) {
+    // A type word riding beside the named object ("what uses the Store
+    // CLASS") describes the OBJECT's grain — it says which "Store" is meant,
+    // not what class of thing may answer. Reading it as the result filter
+    // turned "what uses the Store class" into "which classes use Store" and
+    // answered "no classes" over a module-level graph. Only a type word on
+    // the wh-side of the verb ("which CLASSES use Store") filters the result.
+    const resultType = entityHit && entityHit.start >= verbHit.end ? null : entityType;
+    return stamp({ shape: "reverse", entityType: resultType, modifier, kind, object: afterText });
+  }
   // "what is a kind of class": when the object is itself an entity-type noun, the
   // entity match swallows the whole post-verb span as a grain qualifier, leaving
   // afterText empty — re-read that span as the object instead.

package/src/domain/memory/trust.mjs

@@ -33,6 +33,9 @@ function parseChatTagRest(rest) {
  *   ace:chat:<session>@<ts>    -> { kind:"operator",  createdAt:<ts>, sessionId:<session> }
  *   teach:chat:<session>@<ts>  -> { kind:"teach",     createdAt:<ts>, sessionId:<session> }
  *   web:<url> | url:<url>      -> { kind:"web",       url:<url> }
+ *   reference:<pack>:<article>[@revid] -> { kind:"reference", pack, article }
+ *     (split on the first two colons only; the article keeps any @revid and
+ *      any spaces — "reference:simplewiki:Polar bear@912" stays one article)
  *   extracted:<file-basename>  -> { kind:"extracted", name:<file-basename> }
  *   entailed:<rule>            -> { kind:"entailed",  rule:<rule> }
  * chat:/session: refs map to the operator; an unknown tag -> null (no Source).
@@ -40,6 +43,14 @@ function parseChatTagRest(rest) {
 export function provenanceTagToSource(tag) {
   const t = String(tag || "").trim();
   if (!t) return null;
+  if (t.startsWith("reference:")) {
+    // parsed from the FULL tag, not the whitespace-split head: an article
+    // title may contain spaces ("Polar bear@912").
+    const rest = t.slice("reference:".length);
+    const colon = rest.indexOf(":");
+    if (colon < 0) return { kind: "reference", pack: rest || "unknown", article: "" };
+    return { kind: "reference", pack: rest.slice(0, colon) || "unknown", article: rest.slice(colon + 1) };
+  }
   const head = t.split(/\s+/)[0]; // drop trailing " /r/IsA" etc.
   if (head.startsWith("corpus-weak:")) return { kind: "corpusWeak", name: head.slice("corpus-weak:".length) || "unknown" };
   if (head.startsWith("corpus:")) return { kind: "corpus", name: head.slice("corpus:".length) || "unknown" };
@@ -63,6 +74,7 @@ export const SOURCE_PRIOR = Object.freeze({
   teach: 0.95,
   provider: 0.9,
   corpus: 0.7,
+  reference: 0.6,
   corpusWeak: 0.55,
   web: 0.4,
   extracted: 0.45,

package/src/domain/reference-pack.mjs

@@ -0,0 +1,72 @@
+// reference-pack.mjs — the pure half of the shipped reference pack: the shard
+// naming contract, the index/row shape validators every writer and reader
+// share, the cited answer line, the provenance tag, and the pure part of the
+// clean-miss gate. The pack itself is 64 gzipped JSONL shards plus a gzipped
+// term index; loading them is I/O and lives in
+// src/adapters/corpus/reference-pack.mjs.
+
+import { fnv1aHex, normFactTerm } from "./hash.mjs";
+import { loadLexicon, lookupNoun } from "./grammar/lexicon.mjs";
+import { RELATION_TERM } from "./concept.mjs";
+
+export const REFERENCE_PACK_NAME = "simplewiki";
+export const REFERENCE_SHARD_COUNT = 64;
+
+/** The shard a term's article row lives in: FNV-1a first byte mod 64, as the
+ *  file basename "ref-00" … "ref-3f". Part of the pack's on-disk contract —
+ *  the build script shards with THIS function, so the reader never scans. */
+export function shardNameFor(term) {
+  const byte = parseInt(fnv1aHex(String(term ?? "")).slice(0, 2), 16);
+  return `ref-${(byte % REFERENCE_SHARD_COUNT).toString(16).padStart(2, "0")}`;
+}
+
+const SHARD_NAME_RE = /^ref-[0-3][0-9a-f]$/;
+
+/** An index entry {s, t, r}: the shard holding the row, the row's canonical
+ *  term key (an alias entry points at its target's), and the revision id. */
+export function isReferenceIndexEntry(e) {
+  return !!e && typeof e === "object"
+    && typeof e.s === "string" && SHARD_NAME_RE.test(e.s)
+    && typeof e.t === "string" && e.t.length > 0
+    && Number.isInteger(e.r) && e.r > 0;
+}
+
+/** A shard row: {term, title, text, summary, url, revid, isa?}. */
+export function isReferenceArticleRow(row) {
+  if (!row || typeof row !== "object") return false;
+  for (const field of ["term", "title", "text", "summary", "url"]) {
+    if (typeof row[field] !== "string" || !row[field]) return false;
+  }
+  if (!Number.isInteger(row.revid) || row.revid <= 0) return false;
+  if (row.isa !== undefined && (typeof row.isa !== "string" || !row.isa)) return false;
+  return true;
+}
+
+/** The cited answer for a clean miss the pack could ground: the article's
+ *  summary with its title, licence and revision-pinned URL always visible. */
+export function renderReferenceAnswer(term, article) {
+  return `${term} — ${article.summary} (source: reference article "${article.title}", `
+    + `Simple English Wikipedia, CC BY-SA 4.0 — ${article.url}?oldid=${article.revid})`;
+}
+
+/** The provenance tag a fact stored from a pack article carries —
+ *  memory/trust.mjs parses it back to { kind:"reference", pack, article }. */
+export function referenceProvenanceTag(article) {
+  return `reference:${REFERENCE_PACK_NAME}:${article.title}@${article.revid}`;
+}
+
+/** The pure half of the clean-miss gate: is this term even a pack-lookup
+ *  candidate, and under which key? Returns the pack key (the lexicon entry's
+ *  lemma, normFactTerm-folded) or null when the term is empty, not plain
+ *  words, a vague relation touch (RELATION_TERM owns those), or not in the
+ *  lexicon at all — an unknown word is never a clean miss. The async half of
+ *  the gate (parse succeeded, graph/memory genuinely empty on the term) stays
+ *  with the caller. */
+export function cleanMissReferenceTerm(term, lexicon = loadLexicon()) {
+  const t = normFactTerm(term);
+  if (!t || !/^[a-z][a-z' -]*$/.test(t)) return null;
+  if (RELATION_TERM[t]) return null;
+  const entry = lookupNoun(lexicon, t);
+  if (!entry) return null;
+  return normFactTerm(entry.lemma);
+}

package/src/domain/skos-view.mjs

@@ -0,0 +1,111 @@
+// A derived SKOS concept-identity view over the store's bare-string terms.
+//
+// The corpus stores its term relations as reified facts whose subject/object are
+// bare, normalised strings ("module", not "tmct:module") — normFactTerm strips
+// the CURIE prefix at write time. SKOS S19/S20 fix the domain and range of
+// skos:related to skos:Concept, so the relations cannot map onto SKOS while the
+// terms are plain strings with no concept identity.
+//
+// buildSkosConceptView mints that identity WITHOUT touching storage: it reads the
+// fact rows a store already returns, mints one skos:Concept per normalised term
+// (or per synonym-merged group), and re-expresses mgx:synonym and mgx:relatedTo
+// as SKOS. Nothing is written; the same store, read twice, yields the same view.
+// The minted concepts ARE skos:Concepts by construction, so skos:related between
+// them satisfies S19/S20. skos:prefLabel/altLabel carry no domain (S10/S11), so
+// labelling the minted concepts is well-formed SKOS.
+
+import { normFactTerm } from "./hash.mjs";
+
+export const SKOS_NS = "http://www.w3.org/2004/02/skos/core#";
+
+// mgx:antonym is deliberately absent: SKOS has no opposition relation, and
+// asserting skos:related for an antonym would claim the association the corpus
+// denies. It stays mgx: and reads as a bare string — the honest miss.
+export const DEFAULT_RELATION_MAP = {
+  synonym: ["mgx:synonym"],
+  related: ["mgx:relatedTo", "mgx:similarTo"],
+};
+
+export function buildSkosConceptView(rows, { conceptBase = "concept:", relationMap = DEFAULT_RELATION_MAP } = {}) {
+  const synonymPreds = new Set(relationMap.synonym || []);
+  const relatedPreds = new Set(relationMap.related || []);
+
+  const parent = new Map();
+  const ensure = (t) => { if (!parent.has(t)) parent.set(t, t); };
+  const find = (x) => {
+    let r = x;
+    while (parent.get(r) !== r) r = parent.get(r);
+    while (parent.get(x) !== r) { const next = parent.get(x); parent.set(x, r); x = next; }
+    return r;
+  };
+  const union = (a, b) => {
+    const ra = find(a), rb = find(b);
+    if (ra === rb) return;
+    if (ra < rb) parent.set(rb, ra); else parent.set(ra, rb); // smaller string wins — deterministic
+  };
+
+  const relatedRaw = [];
+  for (const r of rows) {
+    const p = r.predicate;
+    if (!synonymPreds.has(p) && !relatedPreds.has(p)) continue;
+    const s = normFactTerm(r.subject), o = normFactTerm(r.object);
+    if (!s || !o) continue;
+    ensure(s); ensure(o);
+    if (synonymPreds.has(p)) union(s, o); else relatedRaw.push({ s, o });
+  }
+
+  const iriFor = (rep) => conceptBase + rep.replace(/ /g, "_");
+  const componentTerms = new Map();
+  for (const t of parent.keys()) {
+    const rep = find(t);
+    if (!componentTerms.has(rep)) componentTerms.set(rep, new Set());
+    componentTerms.get(rep).add(t);
+  }
+
+  const concepts = [];
+  for (const [rep, terms] of componentTerms) {
+    const sorted = [...terms].sort();
+    concepts.push({ id: iriFor(rep), prefLabel: rep, altLabels: sorted.filter((t) => t !== rep) });
+  }
+  concepts.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+
+  const seen = new Set();
+  const related = [];
+  for (const { s, o } of relatedRaw) {
+    const cs = iriFor(find(s)), co = iriFor(find(o));
+    if (cs === co) continue; // a synonym merge collapsed both terms into one concept
+    // NUL-delimited pair key, same idiom as the store's fact ids — spelled as an
+    // escape so this file stays plain text (a literal NUL reads as binary to git)
+    const key = cs < co ? `${cs}\u0000${co}` : `${co}\u0000${cs}`;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    related.push({ subject: cs, object: co });
+  }
+  related.sort((a, b) => (`${a.subject}${a.object}` < `${b.subject}${b.object}` ? -1 : 1));
+
+  const conceptIdForTerm = (term) => {
+    const t = normFactTerm(term);
+    return parent.has(t) ? iriFor(find(t)) : null;
+  };
+  return { concepts, related, conceptIdForTerm, namespace: SKOS_NS };
+}
+
+/** One term's SKOS neighbourhood out of a store's fact rows: its concept, the
+ *  synonym group read as labels, and the skos:related neighbour concepts.
+ *  `synonyms` is the group minus the queried term — the caller-facing "another
+ *  word for X" answer. Null when the term mints no concept (unknown, or no
+ *  synonym/related facts), so the caller keeps its honest miss. */
+export function relatedForTerm(rows, term, options = {}) {
+  const view = buildSkosConceptView(rows, options);
+  const conceptId = view.conceptIdForTerm(term);
+  if (!conceptId) return null;
+  const byId = new Map(view.concepts.map((c) => [c.id, c]));
+  const concept = byId.get(conceptId);
+  const queried = normFactTerm(term);
+  const synonyms = [concept.prefLabel, ...concept.altLabels].filter((label) => label !== queried);
+  const related = view.related
+    .filter((r) => r.subject === conceptId || r.object === conceptId)
+    .map((r) => byId.get(r.subject === conceptId ? r.object : r.subject))
+    .filter(Boolean);
+  return { conceptId, prefLabel: concept.prefLabel, altLabels: concept.altLabels, synonyms, related };
+}