@rudra-js/attested 0.0.0 → 0.5.0

package/src/phrases.ts

@@ -0,0 +1,227 @@
+import { stripInvisible, stripMarks } from './hidden.js';
+
+/** Claims with no value behind them. Stored in the form `normalisePhrasing` produces. */
+export const BANNED_PHRASES: readonly string[] = [
+  'cheap',
+  'cheaper',
+  'cheapest',
+  'affordable',
+  'bargain',
+  'great value',
+  'best value',
+  'low price',
+  'lower price',
+  'lowest price',
+  'best price',
+  'price drop',
+  'price dropped',
+  'just dropped',
+  'price cut',
+  'reduced price',
+  'sale',
+  'marked down',
+  'half price',
+  'half off',
+  'discount',
+  'discounted',
+  'clearance',
+  'free delivery',
+  'free shipping',
+  'free postage',
+  'free returns',
+  'ships free',
+  'fast delivery',
+  'next day',
+  'same day',
+  'overnight delivery',
+  'arrives tomorrow',
+  'delivered tomorrow',
+  'by tomorrow',
+  'in stock',
+  'out of stock',
+  'low stock',
+  'limited stock',
+  'running low',
+  'running out',
+  'back in stock',
+  'restocked',
+  'sold out',
+  'selling out',
+  'almost sold out',
+  'nearly sold out',
+  'selling fast',
+  'going fast',
+  'going quick',
+  'flying off',
+  'while stocks last',
+  'almost gone',
+  'nearly gone',
+  'few left',
+  'last few',
+  'last chance',
+  'limited time',
+  'today only',
+  'ends soon',
+  'hurry',
+  'act fast',
+  'do not miss',
+  "don't miss",
+  'best seller',
+  'best selling',
+  'bestseller',
+  'bestselling',
+  'most popular',
+  'popular pick',
+  'top pick',
+  'top rated',
+  'highly rated',
+  'highest rated',
+  'rated highest',
+  'best rated',
+  'five star',
+  'star rating',
+  'customer favourite',
+  'customer favorite',
+  'money back',
+];
+
+/**
+ * Letters that render as a Latin letter but are not one. Folded so a Cyrillic о in
+ * "in stоck" cannot walk a claim past its own entry on the list. Both the text and
+ * the phrase are folded, so a genuine Cyrillic phrase still matches itself.
+ */
+const CONFUSABLES: Record<string, string> = {
+  а: 'a',
+  в: 'b',
+  е: 'e',
+  к: 'k',
+  м: 'm',
+  н: 'h',
+  о: 'o',
+  р: 'p',
+  с: 'c',
+  т: 't',
+  х: 'x',
+  у: 'y',
+  ѕ: 's',
+  і: 'i',
+  ј: 'j',
+  ԁ: 'd',
+  һ: 'h',
+  ӏ: 'l',
+  α: 'a',
+  ε: 'e',
+  ι: 'i',
+  κ: 'k',
+  ν: 'v',
+  ο: 'o',
+  ρ: 'p',
+  τ: 't',
+  υ: 'u',
+  χ: 'x',
+  ϲ: 'c',
+};
+
+/** A run of whitespace holding one of these is a break the shopper sees. */
+const LINE_BREAK = /[\n\v\f\r\u0085\u2028\u2029]/;
+
+/** One shape for text and phrases, so a hyphen or a stray mark cannot hide a claim. */
+export function normalisePhrasing(text: string): string {
+  // Marks come off after composition, so an accented e keeps its accent while an
+  // overline, which composes with nothing, does not hide `free` from its own entry.
+  const lowered = stripMarks(stripInvisible(text).normalize('NFKC').toLowerCase().normalize('NFC'));
+
+  let folded = '';
+  for (const char of lowered) folded += CONFUSABLES[char] ?? char;
+
+  // A line break survives as itself. Matching reads straight through it, and only
+  // the allowance in verify.ts cares, because it will not forgive across a break.
+  return folded
+    .replace(/[\u2018\u2019]/g, "'")
+    .replace(/[-_\u2010-\u2015]+/g, ' ')
+    .replace(/\s+/g, (run) => (LINE_BREAK.test(run) ? '\n' : ' '))
+    .trim();
+}
+
+const ASCII_WORD = /[a-z0-9]/;
+
+function isGlued(char: string | undefined): boolean {
+  return char !== undefined && ASCII_WORD.test(char);
+}
+
+export interface Span {
+  start: number;
+  /** The last character of the match, not one past it. */
+  end: number;
+}
+
+export interface Indexed {
+  /** The text as it was handed in, which is what spans index and the word gap reads. */
+  text: string;
+  /** The same characters with the gaps dropped, which is what matching runs against. */
+  haystack: string;
+  /** Where each character of `haystack` really sat in `text`. */
+  spots: number[];
+}
+
+/**
+ * Build this once and read every phrase off it. The text must already be normalised.
+ *
+ * Matching runs with every space dropped, because 送料 無料 is 送料無料 to anyone and a
+ * space is free. `spots` carries each character back to where it really sat, so the
+ * word-gap guard still reads the neighbours the shopper sees.
+ */
+export function indexPhrasing(text: string): Indexed {
+  const tight: string[] = [];
+  const spots: number[] = [];
+  for (let index = 0; index < text.length; index += 1) {
+    const char = text[index] ?? '';
+    // A line break reads like a space here, so a claim split over two lines is caught.
+    if (char === ' ' || char === '\n') continue;
+    tight.push(char);
+    spots.push(index);
+  }
+
+  return { text, haystack: tight.join(''), spots };
+}
+
+/** Spans index the text the index was built from. The phrase must already be normalised. */
+// The word-gap rule only guards an ASCII edge, so `cheap` misses `cheapskate`
+// while a Japanese phrase — which has no word gaps to find — still matches.
+export function spansIn(indexed: Indexed, phrase: string): Span[] {
+  const { text, haystack, spots } = indexed;
+
+  const spans: Span[] = [];
+  const needle = phrase.replaceAll(' ', '').replaceAll('\n', '');
+  if (needle.length === 0) return spans;
+
+  const guardStart = ASCII_WORD.test(needle[0] ?? '');
+  const guardEnd = ASCII_WORD.test(needle[needle.length - 1] ?? '');
+
+  let from = 0;
+  for (;;) {
+    const at = haystack.indexOf(needle, from);
+    if (at === -1) return spans;
+
+    const start = spots[at] ?? 0;
+    const end = spots[at + needle.length - 1] ?? 0;
+    const next = text[end + 1];
+    // A trailing s is a plural, not a different word: `discount` catches `discounts`.
+    const plural = next === 's' && !isGlued(text[end + 2]);
+
+    const before = guardStart && isGlued(text[start - 1]);
+    const after = guardEnd && isGlued(next) && !plural;
+    if (!before && !after) spans.push({ start, end });
+
+    from = at + 1;
+  }
+}
+
+/** Both arguments must already be normalised. Spans index the text passed in. */
+export function phraseSpans(text: string, phrase: string): Span[] {
+  return spansIn(indexPhrasing(text), phrase);
+}
+
+export function phraseIn(text: string, phrase: string): boolean {
+  return phraseSpans(text, phrase).length > 0;
+}

package/src/verify.ts

@@ -0,0 +1,223 @@
+import { numeralsIn, supportedValues, type Numeral } from './numerals.js';
+import { BANNED_PHRASES, indexPhrasing, normalisePhrasing, spansIn, type Span } from './phrases.js';
+
+// Two layers, reported apart, because only one of them is a proof.
+
+export type Layer = 'quantity' | 'wording';
+
+export interface Finding {
+  layer: Layer;
+  /** The numeral as the text wrote it, or the phrase as the denylist holds it. */
+  token: string;
+  /** Audit evidence. Names the layer and the token. */
+  reason: string;
+}
+
+export interface LayerReport {
+  supported: boolean;
+  /**
+   * What this layer's verdict is worth. `proof` means every numeral in the text is
+   * a value the host supplied — not that the sentence around it is true.
+   */
+  strength: 'proof' | 'best-effort';
+  /** Numerals found, or phrases screened against. Zero means nothing was checked. */
+  checked: number;
+  /** One per distinct token that failed. */
+  findings: Finding[];
+}
+
+export interface VerifyResult {
+  /** True when both layers are. */
+  supported: boolean;
+  quantity: LayerReport;
+  wording: LayerReport;
+}
+
+export interface Facts {
+  /** In any shape they are written: `39`, `'$1,299.00'`, `'4,8'`. Every numeral in one counts. */
+  values: readonly (string | number | bigint)[];
+  /** Claims to ban on top of the built-in English list, for the host's own language. */
+  bannedPhrases?: readonly string[];
+  /** Wording the shop stands behind. A banned claim inside one of these is not reported. */
+  allowedPhrases?: readonly string[];
+}
+
+export interface FieldResult {
+  field: string;
+  result: VerifyResult;
+}
+
+export interface BatchResult {
+  /** True when every field is, and when the fields read as one carry no banned claim. */
+  supported: boolean;
+  fields: FieldResult[];
+  /** The wording layer over every field joined, so a phrase split across two fields still reads. */
+  acrossFields: LayerReport;
+}
+
+/**
+ * One reading per fact list, keyed on the list itself, because core hands `verify` the
+ * same array again for every field it screens. Held against a copy of what the list
+ * carried: `readonly` on `Facts.values` is a compile-time view of one reference, and a
+ * host holding the array can write to it between two calls.
+ */
+const readings = new WeakMap<object, { held: readonly unknown[]; supported: Set<string> }>();
+
+function unchanged(held: readonly unknown[], values: readonly unknown[]): boolean {
+  if (held.length !== values.length) return false;
+  for (let i = 0; i < held.length; i += 1) {
+    if (held[i] !== values[i]) return false;
+  }
+  return true;
+}
+
+function valuesOf(facts: Facts): Set<string> {
+  const values = facts.values;
+  // A caller outside TypeScript can pass something with no identity to key on.
+  if (!Array.isArray(values)) return supportedValues(values);
+
+  const read = readings.get(values);
+  if (read !== undefined && unchanged(read.held, values)) return read.supported;
+
+  const supported = supportedValues(values);
+  readings.set(values, { held: [...values], supported });
+  return supported;
+}
+
+function phrasesOf(facts: Facts): string[] {
+  const phrases = new Set<string>(BANNED_PHRASES);
+
+  for (const phrase of facts.bannedPhrases ?? []) {
+    const normalised = normalisePhrasing(phrase);
+    if (normalised.length > 0) phrases.add(normalised);
+  }
+  return [...phrases];
+}
+
+function allowedOf(facts: Facts): string[] {
+  const allowed: string[] = [];
+  for (const phrase of facts.allowedPhrases ?? []) {
+    const normalised = normalisePhrasing(phrase);
+    if (normalised.length > 0) allowed.push(normalised);
+  }
+  return allowed;
+}
+
+function reasonFor(numeral: Numeral): string {
+  if (numeral.kind === 'other-numeral') {
+    return `quantity: "${numeral.token}" is a numeric character this layer cannot read, so no fact can back it`;
+  }
+  if (numeral.kind === 'magnitude') {
+    return `quantity: "${numeral.token}" ends in a magnitude mark this layer cannot read, so the value it shows is unchecked`;
+  }
+  return `quantity: "${numeral.token}" is not a value the supplied facts carry`;
+}
+
+function checkQuantities(text: string, supported: Set<string>): LayerReport {
+  const numerals = numeralsIn(text);
+  const findings: Finding[] = [];
+  const reported = new Set<string>();
+
+  for (const numeral of numerals) {
+    let backed = false;
+    for (const form of numeral.forms) {
+      if (supported.has(form)) backed = true;
+    }
+    if (backed || reported.has(numeral.token)) continue;
+
+    reported.add(numeral.token);
+    findings.push({ layer: 'quantity', token: numeral.token, reason: reasonFor(numeral) });
+  }
+
+  return {
+    supported: findings.length === 0,
+    strength: 'proof',
+    checked: numerals.length,
+    findings,
+  };
+}
+
+function standsBehind(allowed: Span[], hit: Span): boolean {
+  for (const span of allowed) {
+    if (span.start <= hit.start && hit.end <= span.end) return true;
+  }
+  return false;
+}
+
+function checkWording(text: string, phrases: string[], allowed: string[]): LayerReport {
+  const normalised = normalisePhrasing(text);
+  const indexed = indexPhrasing(normalised);
+
+  const spans: Span[] = [];
+  for (const phrase of allowed) {
+    for (const span of spansIn(indexed, phrase)) {
+      // The denylist reads through a break and the allowance does not, so both rules
+      // err toward reporting. A model that puts the negation in one block and the
+      // claim in another has written two things the shopper reads apart.
+      if (!normalised.slice(span.start, span.end + 1).includes('\n')) spans.push(span);
+    }
+  }
+
+  const findings: Finding[] = [];
+  for (const phrase of phrases) {
+    // One hit inside allowed wording is forgiven; the same phrase elsewhere is not.
+    let caught = false;
+    for (const hit of spansIn(indexed, phrase)) {
+      if (!standsBehind(spans, hit)) caught = true;
+    }
+    if (!caught) continue;
+
+    findings.push({
+      layer: 'wording',
+      token: phrase,
+      reason: `wording: "${phrase}" is a claim with no value to check, caught by a best-effort denylist`,
+    });
+  }
+
+  return {
+    supported: findings.length === 0,
+    strength: 'best-effort',
+    checked: phrases.length,
+    findings,
+  };
+}
+
+function check(
+  text: string,
+  supported: Set<string>,
+  phrases: string[],
+  allowed: string[],
+): VerifyResult {
+  const quantity = checkQuantities(text, supported);
+  const wording = checkWording(text, phrases, allowed);
+  return { supported: quantity.supported && wording.supported, quantity, wording };
+}
+
+/** Whether one piece of model-written text stands on the facts the host supplied. */
+export function verify(text: string, facts: Facts): VerifyResult {
+  return check(text, valuesOf(facts), phrasesOf(facts), allowedOf(facts));
+}
+
+/** The same check over every field of one product, without repeating the facts. */
+export function verifyFields(fields: Record<string, string>, facts: Facts): BatchResult {
+  const supported = valuesOf(facts);
+  const phrases = phrasesOf(facts);
+  const allowed = allowedOf(facts);
+
+  const results: FieldResult[] = [];
+  const written: string[] = [];
+  let allSupported = true;
+  for (const field of Object.keys(fields)) {
+    const text = fields[field] ?? '';
+    const result = check(text, supported, phrases, allowed);
+    if (!result.supported) allSupported = false;
+    results.push({ field, result });
+    written.push(text);
+  }
+
+  // A card renders its fields next to each other, so `Free` and `delivery on every
+  // order` are one sentence to a shopper even though neither field carries one.
+  const acrossFields = checkWording(written.join(' '), phrases, allowed);
+
+  return { supported: allSupported && acrossFields.supported, fields: results, acrossFields };
+}