@polycode-projects/the-mechanical-code-talker 0.2.0 → 0.4.0

package/corpus/conceptnet/fetch-slice.mjs

@@ -0,0 +1,136 @@
+#!/usr/bin/env node
+// fetch-slice.mjs — regenerate corpus/conceptnet/slice.jsonl from the public
+// ConceptNet API (https://api.conceptnet.io). NOT part of the product path —
+// a maintainer tool, run by hand, results committed.
+//
+//   node corpus/conceptnet/fetch-slice.mjs [outFile]
+//
+// Polite client: strictly sequential, ~1.1s between requests (the public API
+// asks for roughly 1 req/s sustained), exponential backoff on 429/5xx. A full
+// run over the ~80 seed terms takes a few minutes.
+//
+// Filter rules (also documented in README.md here):
+//   - /query?node=/c/en/<term>&other=/c/en — both endpoints English;
+//   - keep only edges whose rel is one of the 34 canonical relations
+//     (src/corpus/conceptnet-map.toml is the same closed set);
+//   - drop en→en edges whose start/end still carry a sense suffix mismatch
+//     (we keep the bare /c/en/<term> and /c/en/<term>/<pos> forms, normalized
+//     to the bare term URI);
+//   - dedupe by (start, rel, end), keeping the higher weight;
+//   - one JSON object per line: {start, rel, end, surfaceText?, weight},
+//     sorted by rel then start then end (deterministic diffs).
+//
+// Output is CC-BY-SA 4.0 (ConceptNet-derived) — see LICENSE-NOTICE.
+
+import { writeFile } from "node:fs/promises";
+import { fileURLToPath } from "node:url";
+import { dirname, join } from "node:path";
+
+export const SEED_TERMS = [
+  // core artifacts
+  "software", "computer", "program", "code", "source_code", "module",
+  "function", "subroutine", "algorithm", "data_structure", "database",
+  "server", "network", "internet", "file", "directory", "memory",
+  "application", "library", "framework", "api", "operating_system",
+  "compiler", "interpreter", "script", "programming_language",
+  // code constructs
+  "variable", "array", "string", "integer", "boolean", "loop", "class",
+  "object", "method", "parameter", "pointer", "stack", "queue", "cache",
+  "thread", "process", "byte", "bit", "binary", "syntax", "logic",
+  // the work
+  "bug", "error", "test", "debug", "crash", "software_bug", "programmer",
+  "computation", "data", "information", "password", "encryption",
+  // version control
+  "repository", "commit", "branch", "merge", "version",
+  // hardware & devices
+  "keyboard", "mouse", "screen", "monitor", "hardware", "cpu", "processor",
+  "disk", "laptop", "smartphone", "robot", "circuit", "chip",
+  // the wider net
+  "email", "website", "browser", "cloud", "protocol", "http", "terminal",
+  "shell", "command", "linux", "unix", "artificial_intelligence",
+  "machine_learning", "virtual_machine",
+];
+
+export const CANONICAL_RELS = new Set([
+  "/r/RelatedTo", "/r/FormOf", "/r/IsA", "/r/PartOf", "/r/HasA", "/r/UsedFor",
+  "/r/CapableOf", "/r/AtLocation", "/r/Causes", "/r/HasSubevent",
+  "/r/HasFirstSubevent", "/r/HasLastSubevent", "/r/HasPrerequisite",
+  "/r/HasProperty", "/r/MotivatedByGoal", "/r/ObstructedBy", "/r/Desires",
+  "/r/CreatedBy", "/r/Synonym", "/r/Antonym", "/r/DistinctFrom",
+  "/r/DerivedFrom", "/r/SymbolOf", "/r/DefinedAs", "/r/MannerOf",
+  "/r/LocatedNear", "/r/HasContext", "/r/SimilarTo",
+  "/r/EtymologicallyRelatedTo", "/r/EtymologicallyDerivedFrom",
+  "/r/CausesDesire", "/r/MadeOf", "/r/ReceivesAction", "/r/ExternalURL",
+]);
+
+// Relations excluded from the committed slice by policy (see README.md):
+// pure-lexical/etymology noise and link-outs — they'd spend the size budget
+// on rows toFacts() can never emit.
+export const FILTERED_RELS = new Set([
+  "/r/EtymologicallyRelatedTo", "/r/EtymologicallyDerivedFrom", "/r/ExternalURL",
+]);
+
+const API = "https://api.conceptnet.io";
+const LIMIT = 100; // edges fetched per seed term
+const PAUSE_MS = 1100; // polite: ~1 req/s sustained
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+/** Normalize a concept URI to its bare English term: /c/en/dog/n → /c/en/dog.
+ *  Returns null for anything that is not an English concept. */
+export const bareEnTerm = (uri) => {
+  const m = /^\/c\/en\/([^/]+)/.exec(String(uri || ""));
+  return m ? `/c/en/${m[1]}` : null;
+};
+
+/** Filter one raw API edge → a slice row, or null when it fails the rules. */
+export function toRow(edge) {
+  const rel = edge?.rel?.["@id"];
+  if (!rel || !CANONICAL_RELS.has(rel) || FILTERED_RELS.has(rel)) return null;
+  const start = bareEnTerm(edge?.start?.["@id"]);
+  const end = bareEnTerm(edge?.end?.["@id"]);
+  if (!start || !end || start === end) return null;
+  const row = { start, rel, end, weight: Number(edge?.weight) || 1 };
+  if (edge?.surfaceText) row.surfaceText = String(edge.surfaceText);
+  return row;
+}
+
+async function fetchTerm(term, { fetchImpl = fetch, log = console.error } = {}) {
+  const url = `${API}/query?node=/c/en/${term}&other=/c/en&limit=${LIMIT}`;
+  for (let attempt = 1, wait = 5000; attempt <= 5; attempt += 1, wait *= 2) {
+    const res = await fetchImpl(url, { headers: { accept: "application/json" } });
+    if (res.ok) return (await res.json()).edges || [];
+    log(`  ${term}: HTTP ${res.status}${attempt < 5 ? `, backing off ${wait / 1000}s` : " — giving up"}`);
+    if (res.status !== 429 && res.status < 500) return [];
+    if (attempt < 5) await sleep(wait);
+  }
+  throw new Error(`ConceptNet API unreachable while fetching "${term}"`);
+}
+
+export async function fetchSlice({ terms = SEED_TERMS, fetchImpl = fetch, log = console.error, pauseMs = PAUSE_MS } = {}) {
+  const byKey = new Map(); // "start rel end" -> row (higher weight wins)
+  for (const term of terms) {
+    const edges = await fetchTerm(term, { fetchImpl, log });
+    let kept = 0;
+    for (const edge of edges) {
+      const row = toRow(edge);
+      if (!row) continue;
+      const key = `${row.start} ${row.rel} ${row.end}`;
+      const prev = byKey.get(key);
+      if (!prev || row.weight > prev.weight) byKey.set(key, row);
+      kept += 1;
+    }
+    log(`  ${term}: ${edges.length} edges, ${kept} kept`);
+    await sleep(pauseMs);
+  }
+  return [...byKey.values()].sort((a, b) =>
+    a.rel.localeCompare(b.rel) || a.start.localeCompare(b.start) || a.end.localeCompare(b.end));
+}
+
+const isMain = process.argv[1] && import.meta.url === new URL(`file://${process.argv[1]}`).href;
+if (isMain) {
+  const out = process.argv[2] || join(dirname(fileURLToPath(import.meta.url)), "slice.jsonl");
+  const rows = await fetchSlice({});
+  const text = rows.map((r) => JSON.stringify(r)).join("\n") + "\n";
+  await writeFile(out, text);
+  console.error(`wrote ${rows.length} assertions (${text.length} bytes) to ${out}`);
+}

package/corpus/conceptnet/filter-dump.mjs

@@ -0,0 +1,89 @@
+#!/usr/bin/env node
+// filter-dump.mjs — regenerate corpus/conceptnet/slice.jsonl from a ConceptNet
+// ASSERTIONS DUMP instead of the API (the route actually used for the
+// committed slice: api.conceptnet.io was hard-down, 502, on 2026-07-04).
+// NOT part of the product path — a maintainer tool.
+//
+//   curl -s https://s3.amazonaws.com/conceptnet/downloads/2019/edges/conceptnet-assertions-5.7.0.csv.gz \
+//     | gunzip -c \
+//     | node corpus/conceptnet/filter-dump.mjs > corpus/conceptnet/slice.jsonl
+//
+// Input: the tab-separated 5.7.0 dump on stdin —
+//   assertionURI \t rel \t start \t end \t {json: weight, surfaceText, …}
+//
+// Filter rules (shared with fetch-slice.mjs; also in README.md):
+//   - start AND end are English concepts (/c/en/…), sense tags stripped
+//     (/c/en/dog/n → /c/en/dog);
+//   - rel is one of the 34 canonical relations, minus the policy-filtered
+//     etymology/ExternalURL ones;
+//   - at least ONE endpoint's bare term is in the ~90-term tech seed list
+//     (fetch-slice.mjs SEED_TERMS);
+//   - dedupe by (start, rel, end), keeping the higher weight;
+//   - budget (~1.4 MB of JSONL), TWO-TIER: assertions whose relation MAPS to
+//     an ACE-OWL pattern (conceptnet-map.toml, ace != "none") are kept first
+//     (weight-descending); ace="none" relations (RelatedTo, Synonym, …) fill
+//     whatever budget remains — they never crowd out seedable facts;
+//   - final order (rel, start, end) for deterministic diffs.
+// Stats land on stderr; the JSONL lands on stdout.
+
+import { createInterface } from "node:readline";
+import { SEED_TERMS, CANONICAL_RELS, FILTERED_RELS, bareEnTerm } from "./fetch-slice.mjs";
+import { loadMap } from "../../src/corpus/conceptnet.mjs";
+
+const MAX_BYTES = 1_400_000; // committed-slice budget (hard cap 1.5 MB)
+const SEEDS = new Set(SEED_TERMS);
+const termOf = (uri) => uri.slice("/c/en/".length);
+
+const byKey = new Map(); // "start rel end" -> row
+let scanned = 0;
+const rl = createInterface({ input: process.stdin, crlfDelay: Infinity });
+for await (const line of rl) {
+  scanned += 1;
+  const cols = line.split("\t");
+  if (cols.length < 5) continue;
+  const rel = cols[1];
+  if (!CANONICAL_RELS.has(rel) || FILTERED_RELS.has(rel)) continue;
+  const start = bareEnTerm(cols[2]);
+  const end = bareEnTerm(cols[3]);
+  if (!start || !end || start === end) continue;
+  if (!SEEDS.has(termOf(start)) && !SEEDS.has(termOf(end))) continue;
+  let info = {};
+  try {
+    info = JSON.parse(cols[4]);
+  } catch {
+    /* a malformed info column loses us weight/surfaceText, not the edge */
+  }
+  const row = { start, rel, end, weight: Number(info.weight) || 1 };
+  if (info.surfaceText) row.surfaceText = String(info.surfaceText);
+  const key = `${start} ${rel} ${end}`;
+  const prev = byKey.get(key);
+  if (!prev || row.weight > prev.weight) byKey.set(key, row);
+}
+
+// budget-trim: mappable relations first, then none-rows — each tier
+// weight-descending, so the strongest seedable facts always survive
+const map = await loadMap();
+const byWeight = (a, b) =>
+  b.weight - a.weight || a.rel.localeCompare(b.rel) || a.start.localeCompare(b.start) || a.end.localeCompare(b.end);
+const all = [...byKey.values()];
+const mappable = all.filter((r) => map.get(r.rel)?.ace !== "none").sort(byWeight);
+const unmappable = all.filter((r) => map.get(r.rel)?.ace === "none").sort(byWeight);
+const kept = [];
+let bytes = 0;
+for (const row of [...mappable, ...unmappable]) {
+  const line = JSON.stringify(row) + "\n";
+  if (bytes + line.length > MAX_BYTES) continue;
+  bytes += line.length;
+  kept.push(row);
+}
+kept.sort((a, b) => a.rel.localeCompare(b.rel) || a.start.localeCompare(b.start) || a.end.localeCompare(b.end));
+
+for (const row of kept) process.stdout.write(JSON.stringify(row) + "\n");
+
+const perRel = new Map();
+for (const r of kept) perRel.set(r.rel, (perRel.get(r.rel) || 0) + 1);
+const keptMappable = kept.filter((r) => map.get(r.rel)?.ace !== "none").length;
+console.error(`scanned ${scanned} dump lines; matched ${all.length} unique en→en seed assertions `
+  + `(${mappable.length} mappable + ${unmappable.length} ace=none); `
+  + `kept ${kept.length} (${keptMappable} mappable + ${kept.length - keptMappable} none) in ${bytes} bytes`);
+for (const [rel, n] of [...perRel.entries()].sort((a, b) => b[1] - a[1])) console.error(`  ${rel}: ${n}`);