@polycode-projects/the-mechanical-code-talker 6.0.14 → 6.0.15

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@polycode-projects/the-mechanical-code-talker",
-  "version": "6.0.14",
+  "version": "6.0.15",
   "private": false,
   "type": "module",
   "description": "The Mechanical Code Talker (tmct) — a tolerant, offline, $0 chat surface that guides you toward precision queries about a software repository. ELIZA/PARRY-style but domain-obsessed with code. No model calls; indexes a repo on request (tmct index) or reads any producer's graph.",

package/src/adapters/corpus/news-sources.mjs

@@ -298,6 +298,30 @@ async function fetchWikimediaFeed(record, gate, { now }) {
   return { items, bytes: jsonByteLength(body) };
 }
 
+// A Hacker News item is a headline and nothing else — the API carries no
+// summary field to fetch — and a headline is rarely a sentence, so nothing in
+// the item grounds on its own. The fixed sentence below states what the site
+// itself did with the story, and it names the headline IN QUOTES: quoted, the
+// headline's own words can never be re-read as a claim of their own, so "Let's
+// take apart your phone" stays a story Hacker News discusses instead of
+// becoming a fact about Hacker News taking a phone apart.
+//
+// "Hackernews" is one word here on purpose. A two-word proper name in subject
+// position reads as noun + verb ("Hacker" doing "News"), and the fact that
+// falls out of that is nonsense.
+const HACKER_NEWS_TITLE_PREFIX_RE = /^(?:show|ask|tell)\s+hn:\s*/i;
+// Past this many words the stored term is a clause, not a name, and the
+// recognizer turns it down — so the sentence is not built at all rather than
+// stored half-read.
+const HACKER_NEWS_HEADLINE_MAX_WORDS = 6;
+
+function hackerNewsSummary(title) {
+  const headline = String(title || "").replace(HACKER_NEWS_TITLE_PREFIX_RE, "").trim();
+  if (!headline || headline.includes(":")) return "";
+  if (headline.split(/\s+/).length > HACKER_NEWS_HEADLINE_MAX_WORDS) return "";
+  return `Hackernews discusses "${headline}".`;
+}
+
 /** topstories.json, then item/<id>.json for the first ten ids, through the
  *  same gate — each item fetch takes its own slot, so the ten round trips
  *  are genuinely paced by the gate's minIntervalMs rather than firing back
@@ -311,11 +335,12 @@ async function fetchHackerNews(record, gate, { now }) {
     const item = await pacedFetchJson(gate, `${record.url}/item/${id}.json`);
     if (!item) continue;
     bytes += jsonByteLength(item);
+    const title = stripMarkup(item.title || "");
     raw.push({
       guid: String(item.id ?? id),
-      title: stripMarkup(item.title || ""),
+      title,
       url: item.url || `https://news.ycombinator.com/item?id=${item.id ?? id}`,
-      summary: "",
+      summary: hackerNewsSummary(title),
       publishedAt: Number.isFinite(item.time) ? new Date(item.time * 1000).toISOString() : "",
     });
   }
@@ -323,6 +348,32 @@ async function fetchHackerNews(record, gate, { now }) {
   return { items, bytes };
 }
 
+// A USGS place names a point by its distance and bearing from a settlement
+// ("25 km ENE of Wana, Pakistan"), or by an offshore bearing ("off the west
+// coast of Vancouver Island, Canada"), or outright ("Western Australia"). The
+// distance and the bearing are a measurement, not a name, so they come off
+// before the place reaches the sentence below: a number-led term can never
+// head a feed card, and "near" already covers the few kilometres dropped.
+const USGS_DISTANCE_PREFIX_RE = /^\s*\d+(?:\.\d+)?\s*km\s+[NSEW]{1,3}\s+of\s+/i;
+const USGS_OFFSHORE_PREFIX_RE = /^\s*off\s+(?:the\s+)?[a-z\s]*?coast\s+of\s+/i;
+
+function usgsPlaceName(place) {
+  return String(place || "")
+    .replace(USGS_DISTANCE_PREFIX_RE, "")
+    .replace(USGS_OFFSHORE_PREFIX_RE, "")
+    .trim();
+}
+
+// One fixed sentence per quake: a bare-noun subject, the verb the event did,
+// and the named place. The magnitude stays where USGS itself put it, in the
+// item's title — a stored fact's subject here is the CLASS "earthquake", so
+// writing the number into this sentence would say something about earthquakes
+// in general rather than about this one.
+function usgsSummary(place) {
+  const name = usgsPlaceName(place);
+  return name ? `An earthquake struck near ${name}.` : "";
+}
+
 async function fetchUsgs(record, gate, { now }) {
   const body = await pacedFetchJson(gate, record.url);
   if (body === null) return null;
@@ -332,7 +383,7 @@ async function fetchUsgs(record, gate, { now }) {
     guid: String(f?.id ?? f?.properties?.detail ?? f?.properties?.url ?? ""),
     title: stripMarkup(f?.properties?.title || ""),
     url: f?.properties?.url || "",
-    summary: stripMarkup(`magnitude ${f?.properties?.mag ?? "unknown"} earthquake near ${f?.properties?.place || "an unreported location"}`),
+    summary: stripMarkup(usgsSummary(f?.properties?.place)),
     publishedAt: Number.isFinite(f?.properties?.time) ? new Date(f.properties.time).toISOString() : "",
   }));
   const items = normalizeFeedItems(record.id, raw, { now });
@@ -347,7 +398,11 @@ function wikinewsArticleOrigin(apiUrl) {
  *  recentchanges list — the smallest query that names what is new without a
  *  category to maintain by hand. */
 async function fetchWikinews(record, gate, { now }) {
-  const url = `${record.url}?action=query&list=recentchanges&rcnamespace=0&rcnewonly=1&rclimit=20&format=json&formatversion=2&origin=*`;
+  // `rctype=new` is the action API's own name for "page creations only". An
+  // earlier `rcnewonly` is not a parameter the API has: it answers with an
+  // "Unrecognized parameter" warning and lists every edit, so a re-edited old
+  // article read as a new story.
+  const url = `${record.url}?action=query&list=recentchanges&rcnamespace=0&rctype=new&rclimit=20&format=json&formatversion=2&origin=*`;
   const body = await pacedFetchJson(gate, url);
   if (body === null) return null;
   if (isNotModified(body)) return { items: [], bytes: 0, notModified: true };

package/src/domain/news-feed.mjs

@@ -240,6 +240,19 @@ const ENTITY_FRAGMENT_LEAD_WORDS = new Set([
 // term; "back" alone is a fine noun.
 const ENTITY_PARTICLE_LEAD_WORDS = new Set(["back", "up", "down", "out", "off", "away", "along", "around"]);
 
+// A pronoun points back at whatever the last clause named, so a multi-word
+// term opening with one is a clause the split lost the subject of. A term
+// ending in a bare auxiliary is the front half of one. Both mirror
+// extract-facts.mjs's own lexical rules, for the same reason the sets above do.
+const ENTITY_PRONOUN_LEAD_WORDS = new Set([
+  "i", "he", "she", "it", "we", "they", "you", "me", "him", "them", "us",
+  "his", "her", "its", "their", "our", "your", "my",
+]);
+const ENTITY_CLITIC_SUFFIX_RE = /['’](?:s|re|ve|ll|d|m)$/;
+const ENTITY_TRAILING_AUXILIARY_WORDS = new Set([
+  "is", "are", "was", "were", "be", "been", "being", "am", "has", "have", "had",
+]);
+
 /** Does `term` read as a thing's name rather than a clause fragment? Bounds
  *  the word count and rejects a leading conjunction, auxiliary or
  *  preposition (test E's condition 3, PLAN_NEWSWORTHINESS.md section 2), plus
@@ -251,7 +264,10 @@ function looksLikeEntityTerm(term) {
   if (words.length > ENTITY_TERM_MAX_WORDS) return false;
   const first = words[0].toLowerCase().replace(/^[^a-z0-9]+/, "");
   if (!first || ENTITY_FRAGMENT_LEAD_WORDS.has(first)) return false;
-  if (words.length > 1 && ENTITY_PARTICLE_LEAD_WORDS.has(first)) return false;
+  if (words.length === 1) return true;
+  if (ENTITY_PARTICLE_LEAD_WORDS.has(first)) return false;
+  if (ENTITY_PRONOUN_LEAD_WORDS.has(first.replace(ENTITY_CLITIC_SUFFIX_RE, ""))) return false;
+  if (ENTITY_TRAILING_AUXILIARY_WORDS.has(words[words.length - 1].toLowerCase())) return false;
   return true;
 }
 
@@ -425,21 +441,29 @@ export function buildTermAdjacency(rows) {
 }
 
 /** Breadth-first over subject/object adjacency from `hub`, exactly `hops`
- *  levels deep, then capped by content-addressed id — so the cap never
- *  depends on `rows`' own order, only on which rows the hop-bounded walk
- *  actually reaches. */
-export function subgraphAround(rows, hub, { hops = NEWS_HUB_HOPS, cap = 60, adjacency = null } = {}) {
+ *  levels deep, then capped: a `priorityIds` row first, then the nearer hop,
+ *  then content-addressed id. The cap never depends on `rows`' own order, only
+ *  on which rows the hop-bounded walk actually reaches and how far out each
+ *  one sits.
+ *
+ *  `priorityIds` is what keeps a card about a term the graph already knows
+ *  thousands of things about from being built out of an arbitrary slice of
+ *  them: a hub like "france" reaches far more rows than the cap, and the one
+ *  report that made it news would otherwise be the row that fell out. */
+export function subgraphAround(rows, hub, { hops = NEWS_HUB_HOPS, cap = 60, adjacency = null, priorityIds = null } = {}) {
   const adj = adjacency ?? buildTermAdjacency(rows);
   const hubTerm = normFactTerm(hub);
   const visited = new Set([hubTerm]);
   let frontier = [hubTerm];
   const collected = new Map();
+  const hopOf = new Map();
   for (let hop = 0; hop < hops; hop += 1) {
     const nextFrontier = new Set();
     for (const term of [...frontier].sort()) {
       for (const idx of adj.byTerm.get(term) ?? []) {
         const row = rows[idx];
         collected.set(row.id, row);
+        if (!hopOf.has(row.id)) hopOf.set(row.id, hop);
         const [s, o] = adj.terms[idx];
         if (!visited.has(s)) nextFrontier.add(s);
         if (!visited.has(o)) nextFrontier.add(o);
@@ -448,7 +472,12 @@ export function subgraphAround(rows, hub, { hops = NEWS_HUB_HOPS, cap = 60, adja
     for (const term of nextFrontier) visited.add(term);
     frontier = [...nextFrontier].sort();
   }
-  return [...collected.values()].sort(byId).slice(0, cap);
+  const isPriority = (id) => (priorityIds instanceof Set ? priorityIds.has(id) : Boolean(priorityIds?.includes?.(id)));
+  return [...collected.values()]
+    .sort((a, b) => (isPriority(b.id) - isPriority(a.id))
+      || (hopOf.get(a.id) - hopOf.get(b.id))
+      || byId(a, b))
+    .slice(0, cap);
 }
 
 /** The strongest prior kind among `rows`, for the item's trust chip — read
@@ -487,6 +516,28 @@ function joinWithAnd(items) {
 
 const IDENTITY_PREDICATES = new Set(["rdf:type", "rdfs:subClassOf"]);
 const SENTENCE_CAP = 5;
+// How many objects one sentence names before it counts the rest. A live source
+// reports the same relation over and over inside one window — every quake of
+// the day strikes near somewhere — and an unbounded list turns a card into a
+// wall of text.
+const OBJECTS_PER_SENTENCE = 6;
+
+function joinObjects(objects) {
+  if (objects.length <= OBJECTS_PER_SENTENCE) return joinWithAnd(objects);
+  const shown = objects.slice(0, OBJECTS_PER_SENTENCE);
+  return `${shown.join(", ")} and ${objects.length - OBJECTS_PER_SENTENCE} more`;
+}
+
+/** The predicates `rows` carry, curated-table order first and then whatever
+ *  is left, sorted. A relation minted from a source's own verb ("mgx:hit",
+ *  "mgx:strike-near") has no curated entry, and reading the table alone left
+ *  every card built from live headlines with an empty paragraph. */
+function predicatesInRenderOrder(rows) {
+  const present = new Set(rows.map((r) => r.predicate));
+  const curated = Object.keys(FACT_PREDICATE_PHRASES).filter((predicate) => present.has(predicate));
+  const rest = [...present].filter((predicate) => !Object.hasOwn(FACT_PREDICATE_PHRASES, predicate)).sort();
+  return [...curated, ...rest];
+}
 
 /** The fixed five-sentence paraphrase template (PLAN_NEWS_FEED.md section
  *  8.3): identity first, then the hub's own relations grouped by predicate in
@@ -519,17 +570,30 @@ export function renderNewsParagraph(hub, subgraphRows, { reportedIds = null } =
     .sort();
   if (identityObjects.length) {
     const withArticles = identityObjects.map((object) => `${articleFor(object)} ${object}`);
-    sentences.push(`${hub} is ${joinWithAnd(withArticles)}`);
+    sentences.push(`${hub} is ${joinObjects(withArticles)}`);
   }
 
-  for (const predicate of Object.keys(FACT_PREDICATE_PHRASES)) {
+  for (const predicate of predicatesInRenderOrder(reportedHubRows)) {
     if (IDENTITY_PREDICATES.has(predicate) || sentences.length >= SENTENCE_CAP) continue;
     const objects = reportedHubRows
       .filter((r) => r.predicate === predicate)
       .map((r) => r.object)
       .sort();
     if (!objects.length) continue;
-    sentences.push(`${hub} ${predicatePhrase(predicate)} ${joinWithAnd(objects)}`);
+    sentences.push(`${hub} ${predicatePhrase(predicate)} ${joinObjects(objects)}`);
+  }
+
+  // A hub that only ever appears as an OBJECT — the place a quake struck, the
+  // story a site discussed — has no subject-side row to build a sentence from,
+  // and its card came out blank. What was reported about it still says
+  // something, so those rows render whole, subject and all.
+  if (!sentences.length) {
+    const aboutHub = subgraphRows
+      .filter((r) => normFactTerm(r.object) === hubTerm && normFactTerm(r.subject) !== hubTerm && isReported(r.id))
+      .sort(byId)
+      .slice(0, OBJECTS_PER_SENTENCE)
+      .map((r) => factSentence(r));
+    if (aboutHub.length) sentences.push(aboutHub.join("; "));
   }
 
   if (sentences.length < SENTENCE_CAP && secondHopRows.length) {
@@ -564,7 +628,7 @@ export function buildNewsItems(rows, { now, windowMs, limit = 6, sourcesByFactId
   if (readsAsEntityTerm) hubOptions.readsAsEntityTerm = readsAsEntityTerm;
   const hubs = newsworthyHubs(rows, reported, hubOptions);
   const items = hubs.map(({ term, changed }) => {
-    const subgraphRows = subgraphAround(rows, term, { adjacency });
+    const subgraphRows = subgraphAround(rows, term, { adjacency, priorityIds: reportedIds });
     const factIds = subgraphRows.map((r) => r.id).sort();
     const { background } = splitCardRows(subgraphRows, reportedIds);
     return {

package/src/domain/term-ledger.mjs

@@ -8,6 +8,33 @@ import { normFactTerm } from "./hash.mjs";
 
 const ITEM_IDS_CAP = 12;
 
+// A preposition, conjunction or degree/frequency adverb scaffolds a
+// sentence; it never names the thing the sentence is about, so no
+// occurrence count makes one a useful ledger term. Closed set, checked
+// against the already-normalized (lowercased) term. bumpTerms uses this to
+// keep a function word from ever being admitted; ledgerFromPayload uses it
+// to drop one that reached a persisted payload before this filter existed.
+const FUNCTION_WORD_TERMS = new Set([
+  // prepositions
+  "about", "above", "across", "after", "against", "along", "among", "around",
+  "at", "before", "behind", "below", "beneath", "beside", "between", "beyond",
+  "by", "despite", "down", "during", "except", "for", "from", "in", "into",
+  "near", "of", "off", "on", "onto", "out", "over", "since", "through",
+  "throughout", "to", "toward", "towards", "under", "underneath", "until",
+  "up", "upon", "with", "within", "without",
+  // conjunctions
+  "and", "or", "nor", "but", "so", "yet", "although", "because", "if",
+  "though", "unless", "when", "whenever", "whereas", "while", "than",
+  // degree and frequency adverbs
+  "very", "quite", "rather", "too", "just", "only", "even", "also", "still",
+  "already", "almost", "always", "never", "ever", "often", "sometimes",
+  "usually", "indeed", "however", "therefore", "thus", "hence", "meanwhile",
+]);
+
+function isFunctionWordTerm(term) {
+  return FUNCTION_WORD_TERMS.has(term);
+}
+
 /** Ledger entry field order fixed once here so `ledgerPayload` serializes
  *  byte-identically across peers regardless of insertion order elsewhere. */
 function newEntry(term, vocabGrounded, now) {
@@ -35,7 +62,7 @@ export function createTermLedger() {
 export function bumpTerms(ledger, termCounts, itemId, now, vocabGroundedByTerm = new Map()) {
   for (const [rawTerm, occurrences] of termCounts) {
     const term = normFactTerm(rawTerm);
-    if (!term) continue;
+    if (!term || isFunctionWordTerm(term)) continue;
     let entry = ledger.terms.get(term);
     if (!entry) {
       const vocabGrounded = vocabGroundedByTerm.has(rawTerm)
@@ -112,6 +139,7 @@ export function ledgerPayload(ledger) {
 export function ledgerFromPayload(payload) {
   const ledger = createTermLedger();
   for (const entry of payload?.terms ?? []) {
+    if (isFunctionWordTerm(entry.term)) continue;
     ledger.terms.set(entry.term, { ...entry, itemIds: [...(entry.itemIds ?? [])] });
   }
   return ledger;

package/src/services/extract-facts.mjs

@@ -673,6 +673,32 @@ const FRAGMENT_LEAD_TAGS = new Set(["VERB", "AUX", "ADP", "CCONJ", "SCONJ", "PAR
 // which names nothing. Read lexically so a checkout with no wink model catches
 // it too, and only for a multi-word term — "back" alone is a fine noun.
 const PARTICLE_LEAD_WORDS = new Set(["back", "up", "down", "out", "off", "away", "along", "around"]);
+// A pronoun names nothing on its own — it points back at whatever the last
+// clause named — so a multi-word term opening with one is a clause the split
+// lost the subject of ("he's also destroyed the city's soul"), never a name. A
+// one-word term is exempt for the same reason the particle rule exempts one:
+// "us" is also how a headline writes the United States.
+const PRONOUN_LEAD_WORDS = new Set([
+  "i", "he", "she", "it", "we", "they", "you", "me", "him", "them", "us",
+  "his", "her", "its", "their", "our", "your", "my",
+]);
+const CLITIC_SUFFIX_RE = /['’](?:s|re|ve|ll|d|m)$/;
+// A term ending in a bare auxiliary is the front half of a clause the split cut
+// ("rooms were"), never the whole of a name.
+const TRAILING_AUXILIARY_WORDS = new Set([
+  "is", "are", "was", "were", "be", "been", "being", "am", "has", "have", "had",
+]);
+// A compass word opening a place name is a modifier, not a clause lead —
+// "north korea", "south sandwich islands". A tagger reading the LOWERCASED
+// term has no capital left to tell the place from the direction and tags
+// "north"/"south" as an adverb, so the POS rule below would turn every one of
+// them down. Followed by "of" the word really is heading a prepositional
+// phrase ("north of the border"), and that stays declined.
+const COMPASS_LEAD_WORDS = new Set([
+  "north", "south", "east", "west",
+  "northeast", "northwest", "southeast", "southwest",
+  "northern", "southern", "eastern", "western",
+]);
 
 /** Does `term` read as a thing's name rather than a clause fragment? Bounds
  *  the word count and rejects a leading conjunction, auxiliary, preposition,
@@ -689,6 +715,9 @@ export function readsAsEntityTerm(term, nlp) {
   if (FRAGMENT_LEAD_WORDS.has(first)) return false;
   if (words.length === 1) return true;
   if (PARTICLE_LEAD_WORDS.has(first)) return false;
+  if (PRONOUN_LEAD_WORDS.has(first.replace(CLITIC_SUFFIX_RE, ""))) return false;
+  if (TRAILING_AUXILIARY_WORDS.has(words[words.length - 1].toLowerCase())) return false;
+  if (COMPASS_LEAD_WORDS.has(first) && words[1].toLowerCase() !== "of") return true;
   const engine = nlp === undefined ? winkInstance() : nlp;
   if (!engine) return true;
   try {