@polycode-projects/the-mechanical-code-talker 0.5.0 → 0.7.0

package/corpus/conceptnet/filter-dump.mjs

@@ -30,8 +30,65 @@ import { createInterface } from "node:readline";
 import { SEED_TERMS, CANONICAL_RELS, FILTERED_RELS, bareEnTerm } from "./fetch-slice.mjs";
 import { loadMap } from "../../src/corpus/conceptnet.mjs";
 
-const MAX_BYTES = 1_400_000; // committed-slice budget (hard cap 1.5 MB)
-const SEEDS = new Set(SEED_TERMS);
+const MAX_BYTES = 4_500_000; // committed-slice budget (hard cap 5 MB), grown for the ~40k tier-1 target
+
+// Tier-1 growth seeds (2026-07-05): the original SEED_TERMS in fetch-slice.mjs
+// stay the canonical ~90-term base; this list WIDENS the tech domain toward the
+// ~40k-fact target without leaving the software/tech world — languages,
+// frameworks, data structures, cloud/infra, protocols, tools, ML. Kept here (an
+// owned maintainer file) rather than in fetch-slice.mjs so the API route's seed
+// contract is untouched. Union with SEED_TERMS below.
+const EXTRA_SEEDS = [
+  // programming languages & ecosystems
+  "python", "java", "javascript", "typescript", "ruby", "perl", "php", "golang",
+  "rust", "kotlin", "swift", "scala", "haskell", "lisp", "clojure", "erlang",
+  "fortran", "cobol", "pascal", "assembly", "sql", "html", "css", "json", "xml",
+  "yaml", "markdown", "bash", "powershell",
+  // paradigms & concepts
+  "programming", "coding", "recursion", "iteration", "inheritance",
+  "polymorphism", "abstraction", "encapsulation", "concurrency", "parallelism",
+  "multithreading", "asynchronous", "synchronization", "serialization",
+  "optimization", "refactoring", "debugging", "compilation", "runtime",
+  "object_oriented", "functional_programming", "computer_science",
+  "software_engineering", "computer_programming",
+  // data structures & algorithms
+  "hash", "hashtable", "tree", "graph", "list", "tuple", "dictionary", "heap",
+  "matrix", "vector", "node", "recursion", "sorting", "searching", "linked_list",
+  "binary_tree", "hash_table", "data_type", "datatype",
+  // frameworks, tools & platforms
+  "git", "github", "docker", "kubernetes", "jenkins", "react", "angular",
+  "nodejs", "django", "flask", "spring", "rails", "tensorflow", "pytorch",
+  "hadoop", "kafka", "redis", "mongodb", "mysql", "postgresql", "sqlite",
+  "elasticsearch", "apache", "nginx", "wordpress", "eclipse", "vim", "emacs",
+  // hardware & systems
+  "kernel", "driver", "register", "transistor", "semiconductor", "motherboard",
+  "microprocessor", "microcontroller", "gpu", "ram", "rom", "ssd", "firmware",
+  "bootloader", "filesystem", "partition", "peripheral", "microchip",
+  "circuit_board", "integrated_circuit", "hard_drive", "graphics_card",
+  // networking & protocols
+  "packet", "bandwidth", "latency", "router", "firewall", "gateway", "proxy",
+  "dns", "tcp", "ftp", "smtp", "ssh", "ssl", "vpn", "ethernet", "wifi",
+  "bluetooth", "socket", "port", "packet_switching", "ip_address",
+  // web & the net
+  "url", "cookie", "session", "webpage", "hyperlink", "frontend", "backend",
+  "webserver", "webservice", "hosting", "domain", "html5", "ajax",
+  // cloud, infra & devops
+  "cloud_computing", "container", "virtualization", "serverless",
+  "microservice", "deployment", "devops", "pipeline", "infrastructure",
+  "datacenter", "cluster", "scalability", "availability", "redundancy",
+  // security
+  "authentication", "authorization", "cryptography", "hashing", "firewall",
+  "malware", "virus", "vulnerability", "cybersecurity", "cipher",
+  // data & ml
+  "dataset", "database", "datastore", "query", "index", "schema", "transaction",
+  "neural_network", "deep_learning", "classification", "regression",
+  "clustering", "training", "inference", "algorithm", "computation",
+  "big_data", "data_mining", "data_science", "analytics",
+  // concurrency & os primitives
+  "semaphore", "mutex", "deadlock", "scheduler", "interrupt", "syscall",
+  "daemon", "multitasking", "buffer", "pipe",
+];
+const SEEDS = new Set([...SEED_TERMS, ...EXTRA_SEEDS]);
 const termOf = (uri) => uri.slice("/c/en/".length);
 
 const byKey = new Map(); // "start rel end" -> row

package/corpus/conceptnet/quality-filter.mjs

@@ -0,0 +1,95 @@
+#!/usr/bin/env node
+// quality-filter.mjs — a SECOND-PASS noise filter over corpus/conceptnet/slice.jsonl.
+// NOT part of the product path — a maintainer tool, run by hand, result committed.
+//
+// The committed slice is stream-filtered from the ConceptNet dump by
+// filter-dump.mjs (tech-domain seed match + canonical relations + budget). That
+// pass keeps the DATA honest but not the SEMANTICS: ConceptNet's crowd-sourced
+// "Verbosity"/Open-Mind rows leave sentence-fragment "concepts" and opinion
+// axioms in the slice that read as nonsense once they become memory facts —
+// e.g. "a computer is a kind of dumb", "a class is a kind of elegance",
+// "mouse AtLocation taloned_grip_of_owl", "2 is a kind of software".
+//
+// This pass removes exactly those, by term/relation shape only (never by hand
+// per row), so it is reproducible:
+//
+//   node corpus/conceptnet/quality-filter.mjs < corpus/conceptnet/slice.jsonl > slice.clean.jsonl
+//   # or in place (what produced the committed clean slice):
+//   node corpus/conceptnet/quality-filter.mjs --in-place corpus/conceptnet/slice.jsonl
+//
+// Cut rules (a row is DROPPED when ANY applies):
+//   1. numeric endpoint      — start/end bare term is all digits ("2", "1000")
+//   2. single-char endpoint  — bare term length <= 1 ("a", "r", "m")
+//   3. sentence fragment     — bare term is >= 4 underscore-words on EITHER
+//                              endpoint ("taloned_grip_of_owl",
+//                              "worlds_largest_interconnected_network_of_networks")
+//   4. definitional phrase   — /r/DefinedAs whose object is >= 3 words
+//                              (real DefinedAs is a synonym: cpu->processor)
+//   5. opinion object        — /r/IsA or /r/DefinedAs whose object is one of a
+//                              small, evidence-based OPINION set (adjectives /
+//                              value words that never name a class)
+//
+// Rules 1-3 apply to every relation (they only ever remove junk); 4-5 are the
+// definitional band the sims flagged. Stats land on stderr; JSONL on stdout.
+
+import { readFile, writeFile } from "node:fs/promises";
+import { createInterface } from "node:readline";
+
+const bareTerm = (uri) => String(uri || "").replace(/^\/c\/en\//, "");
+const words = (t) => t.split("_").filter(Boolean).length;
+
+// Evidence-based: the only 1-word IsA/DefinedAs objects in the committed slice
+// that are opinions/adjectives rather than classes. Kept explicit (not a POS
+// heuristic) so it never cuts a legitimate abstract class like "abstraction",
+// "cognition" or "relation".
+export const OPINION_OBJECTS = new Set([
+  "elegance", "evil", "gloom", "unreality", "universalism", "dumb", "free", "junk",
+]);
+
+/** Why this row is noise, or null to keep it. Pure function of the row shape. */
+export function cutReason(row) {
+  const s = bareTerm(row.start);
+  const e = bareTerm(row.end);
+  for (const t of [s, e]) {
+    if (/^\d+$/.test(t)) return "numeric-endpoint";
+    if (t.length <= 1) return "single-char-endpoint";
+    if (words(t) >= 4) return "sentence-fragment";
+  }
+  if (row.rel === "/r/DefinedAs" && words(e) >= 3) return "definitional-phrase";
+  if ((row.rel === "/r/IsA" || row.rel === "/r/DefinedAs") && OPINION_OBJECTS.has(e)) return "opinion-object";
+  return null;
+}
+
+async function readLines(stream) {
+  const rl = createInterface({ input: stream, crlfDelay: Infinity });
+  const rows = [];
+  for await (const raw of rl) {
+    const line = raw.trim();
+    if (line) rows.push(JSON.parse(line));
+  }
+  return rows;
+}
+
+const isMain = process.argv[1] && import.meta.url === new URL(`file://${process.argv[1]}`).href;
+if (isMain) {
+  const inPlace = process.argv.includes("--in-place");
+  const fileArg = process.argv.slice(2).find((a) => !a.startsWith("--"));
+  const rows = inPlace
+    ? (await readFile(fileArg, "utf8")).split("\n").filter((l) => l.trim()).map((l) => JSON.parse(l))
+    : await readLines(process.stdin);
+
+  const byReason = new Map();
+  const kept = [];
+  for (const row of rows) {
+    const reason = cutReason(row);
+    if (reason) { byReason.set(reason, (byReason.get(reason) || 0) + 1); continue; }
+    kept.push(row);
+  }
+  const text = kept.map((r) => JSON.stringify(r)).join("\n") + "\n";
+  if (inPlace) await writeFile(fileArg, text);
+  else process.stdout.write(text);
+
+  const cut = rows.length - kept.length;
+  console.error(`quality-filter: ${rows.length} rows in, ${kept.length} kept, ${cut} cut (${text.length} bytes out)`);
+  for (const [reason, n] of [...byReason.entries()].sort((a, b) => b[1] - a[1])) console.error(`  ${reason}: ${n}`);
+}