@hviana/sema 0.5.9 → 0.7.1

package/dist/example/train_base.d.ts

@@ -1,163 +0,0 @@
-export interface Episode {
-    context: string;
-    continuation: string;
-}
-export type TrainingItem = string | Episode;
-/** Dedup + trim a concept's items: drop empty/degenerate pairs and exact
- *  repeats so a concept never deposits the same form twice. */
-export declare function refineItems(items: TrainingItem[]): TrainingItem[];
-/** One normalized SmolSent row. */
-export interface SmolSentRow {
-    src: string;
-    trg: string;
-    sl: string;
-    tl: string;
-}
-/** Normalize a raw datasets-server row into a SmolSentRow, or null when it lacks
- *  both sides or a side is implausibly large (a dump, not a sentence). */
-export declare function toSmolSentRow(row: unknown): SmolSentRow | null;
-/** Translate ONE SmolSent pair into SEMA facts. The two sentences are one
- *  meaning in two languages, but the two BINDINGS are not equally sound —
- *  SmolSent's English side is a shared pool translated into every language, so
- *  `trg -> src` gives one English context a different answer in every language
- *  file. See SMOLSENT_DIRECTIONS. refineItems drops the degenerate case where
- *  src === trg. */
-export declare function smolSentRowToItems(row: SmolSentRow): TrainingItem[];
-/** One normalized Aya row. */
-export interface AyaRow {
-    inputs: string;
-    targets: string;
-    language: string;
-}
-/** Normalize a raw datasets-server row object into an AyaRow, or null when it
- *  lacks a usable prompt/answer or a field is implausibly large (a dump, not a
- *  cognitive example). Trims surrounding whitespace; keeps inner text verbatim
- *  (human prose, possibly multi-paragraph). */
-export declare function toAyaRow(row: unknown): AyaRow | null;
-/** Translate ONE Aya row into SEMA training items. A row is a single human
- *  (question → answer) exchange — exactly one FACT, the (inputs → targets) edge.
- *  No standalone-answer experience and no one-exchange "cumulative" walk: a lone
- *  Q→A is not multi-turn, and both would only replicate the same edge. */
-export declare function ayaRowToItems(row: AyaRow): TrainingItem[];
-/** A single oasst2 message node (the fields we use; the tree nests via replies). */
-interface OasstNode {
-    role?: string;
-    text?: string;
-    rank?: number | null;
-    deleted?: boolean;
-    replies?: OasstNode[];
-}
-/** One conversational turn extracted from a tree. */
-export interface OasstTurn {
-    role: string;
-    text: string;
-}
-/** Collapse a conversation tree to ONE linear path: at each node, descend into
- *  its best-ranked, non-deleted reply (rank 0 preferred; unranked sorts last).
- *  Returns the ordered turns (already strictly alternating in this corpus). */
-export declare function bestOasstPath(root: OasstNode): OasstTurn[];
-/** Translate ONE multi-turn oasst2 conversation into SEMA training items.
- *
- *  This is the ONE stage where cumulative continuous context is truly necessary:
- *  the data is a real multi-turn dialogue, and what must be learned is how each
- *  turn follows from the WHOLE conversation so far — not from the previous turn
- *  alone. The conversation is emitted ONLY as the accumulated walk; standalone
- *  turn experiences and local adjacent-pair facts are NOT emitted (they are
- *  subsumed by it and would merely replicate the content).
- *
- *  The walk is the pattern proven in test/13-conversation.test.mjs
- *  ("teachConversation"): each turn is the continuation of all prior turns,
- *  with BARE turn text — NO "User:/Assistant:" labels.  The SHAPE is identical
- *  (cumulative context → next turn); the join string is not, and does not need
- *  to be — that file joins with nothing and this corpus joins with "\n" (see
- *  `accumulate`).  Saying "byte-for-byte", as this comment used to, invites the
- *  reading that the two must agree on a separator.  They must not agree,
- *  because there is nothing to agree about: turn boundaries are offsets, and
- *  the join string is just corpus text. Roles already
- *  alternate by position in an oasst2 best-path (the root is a prompter), so a
- *  label adds nothing the position does not, while a clean continuation matches
- *  the test's recall (predictNext queries bare prior turns) and lets a turn share
- *  its gist with the same text elsewhere (e.g. an Aya question stored bare).
- *
- *  Returns [] for a conversation below the multi-turn threshold, so callers can
- *  simply skip empties. */
-export declare function oasstConversationToItems(turns: OasstTurn[]): TrainingItem[];
-/** One utterance of a Taskmaster conversation. */
-export interface TaskmasterTurn {
-    speaker: string;
-    text: string;
-}
-/** Normalize ONE element of a Taskmaster data file into its turns, or null when
- *  it carries no usable utterance. Empty/whitespace-only utterances are dropped
- *  (TM-3 has a few); a single implausibly long utterance rejects the whole
- *  conversation as corrupt rather than depositing a dump. */
-export declare function toTaskmasterTurns(row: unknown): TaskmasterTurn[] | null;
-/** Collapse consecutive same-speaker turns into one, joining with a space, and
- *  return the bare texts in order. A turn with no speaker never merges with its
- *  neighbour: an unlabelled row is of unknown origin, and joining two of them
- *  would invent a contribution that may span two speakers. */
-export declare function mergeTaskmasterTurns(turns: TaskmasterTurn[]): string[];
-/** Translate ONE Taskmaster conversation into SEMA training items: the
- *  cumulative walk over its merged turns. Returns [] for a conversation below
- *  TASKMASTER_MIN_TURNS, so callers can simply skip empties. */
-export declare function taskmasterConversationToItems(turns: TaskmasterTurn[]): TrainingItem[];
-/** One (subject, relation, object) triple from a 2Wiki `evidences` cell. */
-export interface WikiTriple {
-    subject: string;
-    relation: string;
-    object: string;
-}
-/** Normalize a 2Wiki row into its evidence triples, or null when it carries
- *  none usable. `evidences` is a JSON STRING holding an array of 3-element
- *  arrays; a row whose cell is absent, unparseable, or empty yields null.
- *  Individual malformed or oversized triples are dropped without discarding the
- *  row — one bad triple should not cost the others. */
-export declare function toWikiTriples(row: unknown): WikiTriple[] | null;
-/** Render ONE triple as the prose fact Sema stores. Kept separate so the two
- *  deposits below are guaranteed to share a byte-identical continuation: the
- *  pivot fact only works if it leads to the SAME node the relation fact does. */
-export declare function wikiTripleSentence(t: WikiTriple): string;
-/** Translate a row's triples into SEMA items: per triple, the relation fact and
- *  the bare-subject PIVOT fact (see the section note above). refineItems drops
- *  the duplicates this produces when a row states the same triple twice. */
-export declare function wikiTriplesToItems(triples: WikiTriple[]): TrainingItem[];
-/** Normalize a SODA row into its turns, or null when it carries no usable
- *  dialogue. Speakers are optional (they only drive merging); an implausibly
- *  long turn rejects the dialogue as corrupt. */
-export declare function toSodaTurns(row: unknown): TaskmasterTurn[] | null;
-/** Translate ONE SODA dialogue into SEMA items: the cumulative walk over its
- *  speaker-merged turns. Shares `mergeTaskmasterTurns` because the rule is the
- *  same one — consecutive turns by one speaker are one contribution. */
-export declare function sodaDialogueToItems(turns: TaskmasterTurn[]): TrainingItem[];
-/** Translate ONE MASSIVE row into SEMA items: its bare utterance, as an
- *  experience. `annot_utt` (slot-annotated) is deliberately not used — its
- *  "[date : ...]" markup is not prose. Returns [] for an unusable row. */
-export declare function massiveRowToItems(row: unknown): TrainingItem[];
-/** One normalized General-Knowledge row. */
-export interface GenKnowRow {
-    question: string;
-    answer: string;
-}
-/** Normalize a raw datasets-server row into a GenKnowRow, or null when it lacks
- *  a usable question/answer or a side is implausibly large (corruption). */
-export declare function toGenKnowRow(row: unknown): GenKnowRow | null;
-/** Translate ONE General-Knowledge row into SEMA items: exactly one
- *  (question → answer) FACT. refineItems drops a degenerate question === answer. */
-export declare function genKnowRowToItems(row: GenKnowRow): TrainingItem[];
-/** How many rows to materialise in one read from a row-group of `rgRows` rows
- *  occupying `groupBytes` uncompressed bytes, under a `budgetBytes` target.
- *
- *  The group's own footer statistics give the mean row width, so the batch
- *  follows the CORPUS's row size rather than the writer's layout: wide rows
- *  (SODA carries a whole dialogue per row) batch smaller than narrow ones at
- *  the same memory cost. Never exceeds the group — a batch is a subdivision of
- *  a group, never a span across two, because `parquetReadObjects` is given an
- *  absolute row range and column chunks are per-group. Never returns 0, or the
- *  read loop could not advance.
- *
- *  A writer that omits `total_byte_size` yields `groupBytes <= 0`; the batch is
- *  then the whole group, which is exactly the behaviour this replaced. That
- *  fallback is safe for every file we read today (all three report it) and
- *  degrades to the old memory profile rather than to a wrong result. */
-export declare function parquetBatchRows(rgRows: number, groupBytes: number, budgetBytes: number): number;
-export {};