@hviana/sema 0.5.8 → 0.6.0

package/test/88-dependency-footprint.test.mjs

@@ -0,0 +1,99 @@
+// 88 — the dependency footprint is a PRODUCT PROPERTY, so it is tested.
+//
+// AGENTS.md §6: "do not add runtime dependencies casually — the near-zero-
+// dependency footprint is a product feature." A feature stated only in prose
+// erodes; this suite pins it at the two places it can actually break.
+//
+//   1. The BUILT LIBRARY imports nothing outside `node:`. Not the source — the
+//      built output, because that is what a consumer loads.
+//   2. The PACKAGE declares no runtime `dependencies`. A package can be
+//      import-clean and still force installs on every consumer; that is exactly
+//      how `hyparquet` — used only by example/train_base, never by src/ — came
+//      to be installed by everyone who depended on Sema.
+//
+// The trainer example is deliberately NOT covered: it may use whatever it
+// needs, as a dev dependency, loaded lazily (see example/train_base/readers.ts).
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { readdirSync, readFileSync, statSync } from "node:fs";
+import { join } from "node:path";
+
+const ROOT = new URL("..", import.meta.url).pathname;
+
+/** Every built .js file of the library itself. */
+function libFiles(dir = join(ROOT, "dist/src"), out = []) {
+  for (const name of readdirSync(dir)) {
+    const p = join(dir, name);
+    if (statSync(p).isDirectory()) libFiles(p, out);
+    else if (name.endsWith(".js")) out.push(p);
+  }
+  return out;
+}
+
+/** Module specifiers of static imports/exports and dynamic import() calls. */
+function specifiersOf(src) {
+  const out = [];
+  const patterns = [
+    /(?:^|\s)(?:import|export)\s[^;]*?from\s*["']([^"']+)["']/g,
+    /(?:^|\s)import\s*["']([^"']+)["']/g,
+    /\bimport\s*\(\s*["']([^"']+)["']\s*\)/g,
+    /\brequire\s*\(\s*["']([^"']+)["']\s*\)/g,
+  ];
+  for (const re of patterns) {
+    for (const m of src.matchAll(re)) out.push(m[1]);
+  }
+  return out;
+}
+
+test("the built library imports nothing outside node: builtins", () => {
+  const files = libFiles();
+  assert.ok(files.length > 0, "dist/src is empty — run `npm run build` first");
+
+  const foreign = [];
+  for (const file of files) {
+    for (const spec of specifiersOf(readFileSync(file, "utf8"))) {
+      const ok = spec.startsWith("./") || spec.startsWith("../") ||
+        spec.startsWith("node:");
+      if (!ok) foreign.push(`${file.slice(ROOT.length)} → ${spec}`);
+    }
+  }
+  assert.deepEqual(
+    foreign,
+    [],
+    "the library gained a third-party import; keep it in an example instead",
+  );
+});
+
+test("the package forces no runtime dependencies on consumers", () => {
+  const pkg = JSON.parse(
+    readFileSync(join(ROOT, "package.json"), "utf8"),
+  );
+  const deps = Object.keys(pkg.dependencies ?? {});
+  assert.deepEqual(
+    deps,
+    [],
+    "a package a consumer installs must not carry dependencies only an " +
+      "example needs — declare them under devDependencies",
+  );
+});
+
+test("the published entry points resolve inside dist/src", () => {
+  // A `main`/`exports` that reached outside dist/src would drag the example —
+  // and its dev dependencies — into a consumer's module graph.
+  const pkg = JSON.parse(readFileSync(join(ROOT, "package.json"), "utf8"));
+  const entries = [
+    pkg.main,
+    pkg.types,
+    pkg.exports?.["."]?.default,
+    pkg.exports?.["."]?.types,
+  ].filter(Boolean);
+  assert.ok(entries.length > 0, "no entry points declared");
+  for (const e of entries) {
+    assert.match(
+      e,
+      /^(\.\/)?dist\/src\//,
+      `entry point "${e}" escapes dist/src`,
+    );
+  }
+});

package/dist/example/train_base.d.ts

@@ -1,93 +0,0 @@
-export interface Episode {
-    context: string;
-    continuation: string;
-}
-export type TrainingItem = string | Episode;
-/** Dedup + trim a concept's items: drop empty/degenerate pairs and exact
- *  repeats so a concept never deposits the same form twice. */
-export declare function refineItems(items: TrainingItem[]): TrainingItem[];
-/** One normalized SmolSent row. */
-export interface SmolSentRow {
-    src: string;
-    trg: string;
-    sl: string;
-    tl: string;
-}
-/** Normalize a raw datasets-server row into a SmolSentRow, or null when it lacks
- *  both sides or a side is implausibly large (a dump, not a sentence). */
-export declare function toSmolSentRow(row: unknown): SmolSentRow | null;
-/** Translate ONE SmolSent pair into SEMA facts: the two sentences are one
- *  meaning in two languages, so bind them BOTH ways. refineItems drops the
- *  degenerate case where src === trg. */
-export declare function smolSentRowToItems(row: SmolSentRow): TrainingItem[];
-/** One normalized Aya row. */
-export interface AyaRow {
-    inputs: string;
-    targets: string;
-    language: string;
-}
-/** Normalize a raw datasets-server row object into an AyaRow, or null when it
- *  lacks a usable prompt/answer or a field is implausibly large (a dump, not a
- *  cognitive example). Trims surrounding whitespace; keeps inner text verbatim
- *  (human prose, possibly multi-paragraph). */
-export declare function toAyaRow(row: unknown): AyaRow | null;
-/** Translate ONE Aya row into SEMA training items. A row is a single human
- *  (question → answer) exchange — exactly one FACT, the (inputs → targets) edge.
- *  No standalone-answer experience and no one-exchange "cumulative" walk: a lone
- *  Q→A is not multi-turn, and both would only replicate the same edge. */
-export declare function ayaRowToItems(row: AyaRow): TrainingItem[];
-/** A single oasst2 message node (the fields we use; the tree nests via replies). */
-interface OasstNode {
-    role?: string;
-    text?: string;
-    rank?: number | null;
-    deleted?: boolean;
-    replies?: OasstNode[];
-}
-/** One conversational turn extracted from a tree. */
-export interface OasstTurn {
-    role: string;
-    text: string;
-}
-/** Collapse a conversation tree to ONE linear path: at each node, descend into
- *  its best-ranked, non-deleted reply (rank 0 preferred; unranked sorts last).
- *  Returns the ordered turns (already strictly alternating in this corpus). */
-export declare function bestOasstPath(root: OasstNode): OasstTurn[];
-/** Translate ONE multi-turn oasst2 conversation into SEMA training items.
- *
- *  This is the ONE stage where cumulative continuous context is truly necessary:
- *  the data is a real multi-turn dialogue, and what must be learned is how each
- *  turn follows from the WHOLE conversation so far — not from the previous turn
- *  alone. The conversation is emitted ONLY as the accumulated walk; standalone
- *  turn experiences and local adjacent-pair facts are NOT emitted (they are
- *  subsumed by it and would merely replicate the content).
- *
- *  The walk is the pattern proven in test/13-conversation.test.mjs
- *  ("teachConversation"): each turn is the continuation of all prior turns,
- *  with BARE turn text — NO "User:/Assistant:" labels.  The SHAPE is identical
- *  (cumulative context → next turn); the join string is not, and does not need
- *  to be — that file joins with nothing and this corpus joins with "\n" (see
- *  `accumulate`).  Saying "byte-for-byte", as this comment used to, invites the
- *  reading that the two must agree on a separator.  They must not agree,
- *  because there is nothing to agree about: turn boundaries are offsets, and
- *  the join string is just corpus text. Roles already
- *  alternate by position in an oasst2 best-path (the root is a prompter), so a
- *  label adds nothing the position does not, while a clean continuation matches
- *  the test's recall (predictNext queries bare prior turns) and lets a turn share
- *  its gist with the same text elsewhere (e.g. an Aya question stored bare).
- *
- *  Returns [] for a conversation below the multi-turn threshold, so callers can
- *  simply skip empties. */
-export declare function oasstConversationToItems(turns: OasstTurn[]): TrainingItem[];
-/** One normalized General-Knowledge row. */
-export interface GenKnowRow {
-    question: string;
-    answer: string;
-}
-/** Normalize a raw datasets-server row into a GenKnowRow, or null when it lacks
- *  a usable question/answer or a side is implausibly large (corruption). */
-export declare function toGenKnowRow(row: unknown): GenKnowRow | null;
-/** Translate ONE General-Knowledge row into SEMA items: exactly one
- *  (question → answer) FACT. refineItems drops a degenerate question === answer. */
-export declare function genKnowRowToItems(row: GenKnowRow): TrainingItem[];
-export {};