agent-simple-english 0.1.0

package/src/engine/rules/dictionary.ts

@@ -0,0 +1,281 @@
+import { DICTIONARY_TOKEN_SOURCE } from "../../dictionary/form.ts"
+import type { Dictionary, DictionaryEntry } from "../../dictionary/schema.ts"
+import type { TaggedToken, Tagger } from "../tagger.ts"
+import type { Violation } from "../types.ts"
+
+interface WordToken {
+  readonly text: string
+  readonly lower: string
+  readonly lineIndex: number
+  readonly offset: number
+}
+
+interface Form {
+  readonly entry: DictionaryEntry
+  readonly words: readonly string[]
+}
+
+interface MarkdownContext {
+  readonly contentStart: number
+  readonly quoteDepth: number
+  readonly paragraphId?: number
+}
+
+interface ActiveParagraph {
+  readonly id: number
+  readonly quoteDepth: number
+}
+
+const ATX_HEADING = /^ {0,3}#{1,6}(?:[\t ]+|$)/
+const LIST_MARKER = /^ {0,3}(?:[-+*]|\d{1,9}[.)])(?:[\t ]+|$)/
+const SETEXT_UNDERLINE = /^ {0,3}(?:=+|-+)[\t ]*\r?$/
+const THEMATIC_BREAK = /^ {0,3}(?:(?:\*[\t ]*){3,}|(?:_[\t ]*){3,}|(?:-[\t ]*){3,})\r?$/
+
+const tokenize = (lines: readonly string[]): readonly WordToken[] => {
+  const tokenPattern = new RegExp(DICTIONARY_TOKEN_SOURCE, "gu")
+  return lines.flatMap((line, lineIndex) =>
+    Array.from(line.matchAll(tokenPattern), (match) => ({
+      text: match[0],
+      lower: match[0].toLowerCase(),
+      lineIndex,
+      offset: match.index,
+    })),
+  )
+}
+
+const compileForms = (dictionary: Dictionary): readonly Form[] =>
+  dictionary.entries
+    .flatMap((entry) =>
+      entry.unapproved.map((form) => ({ entry, words: form.toLowerCase().split(/\s+/) })),
+    )
+    .sort((left, right) => right.words.length - left.words.length)
+
+const markdownContext = (line: string, initialContentStart = 0): MarkdownContext => {
+  let contentStart = Math.min(initialContentStart, line.length)
+  let quoteDepth = 0
+
+  while (contentStart < line.length) {
+    let marker = contentStart
+    let spaces = 0
+    while (spaces < 4 && line[marker] === " ") {
+      marker++
+      spaces++
+    }
+    if (spaces > 3 || line[marker] !== ">") {
+      break
+    }
+    contentStart = marker + 1
+    if (line[contentStart] === " " || line[contentStart] === "\t") {
+      contentStart++
+    }
+    quoteDepth++
+  }
+
+  return { contentStart, quoteDepth }
+}
+
+const blockContent = (line: string, context: MarkdownContext): string =>
+  line.slice(context.contentStart)
+
+const isLeafBlock = (content: string): boolean => {
+  const listMarker = content.match(LIST_MARKER)
+  const nestedContent = listMarker === null ? content : content.slice(listMarker[0].length)
+  return ATX_HEADING.test(nestedContent)
+}
+
+const startsNewBlock = (content: string): boolean =>
+  ATX_HEADING.test(content) ||
+  LIST_MARKER.test(content) ||
+  SETEXT_UNDERLINE.test(content) ||
+  THEMATIC_BREAK.test(content)
+
+const isParagraphBlock = (content: string): boolean =>
+  !isLeafBlock(content) && !SETEXT_UNDERLINE.test(content) && !THEMATIC_BREAK.test(content)
+
+const isIndentedCode = (content: string): boolean => /^(?: {4}|\t)/.test(content)
+
+const markdownContexts = (
+  lines: readonly string[],
+  contentStarts: readonly number[],
+): readonly MarkdownContext[] => {
+  let activeParagraph: ActiveParagraph | undefined
+  let nextParagraphId = 0
+
+  return lines.map((line, lineIndex) => {
+    const context = markdownContext(line, contentStarts[lineIndex] ?? 0)
+    const content = blockContent(line, context)
+    if (/^[\t ]*\r?$/.test(content)) {
+      activeParagraph = undefined
+      return context
+    }
+
+    if (
+      activeParagraph !== undefined &&
+      context.quoteDepth <= activeParagraph.quoteDepth &&
+      !startsNewBlock(content)
+    ) {
+      return { ...context, paragraphId: activeParagraph.id }
+    }
+
+    if (isIndentedCode(content)) {
+      activeParagraph = undefined
+      return context
+    }
+
+    const paragraphId = nextParagraphId++
+    activeParagraph = isParagraphBlock(content)
+      ? { id: paragraphId, quoteDepth: context.quoteDepth }
+      : undefined
+    return { ...context, paragraphId }
+  })
+}
+
+const isSoftLineBreak = (
+  lines: readonly string[],
+  contexts: readonly MarkdownContext[],
+  previous: WordToken,
+  token: WordToken,
+): boolean => {
+  if (token.lineIndex !== previous.lineIndex + 1) {
+    return false
+  }
+  const previousLine = lines[previous.lineIndex]
+  const nextLine = lines[token.lineIndex]
+  if (previousLine === undefined || nextLine === undefined) {
+    return false
+  }
+
+  const previousContext = contexts[previous.lineIndex]
+  const nextContext = contexts[token.lineIndex]
+  if (
+    previousContext === undefined ||
+    nextContext === undefined ||
+    previousContext.paragraphId === undefined ||
+    previousContext.paragraphId !== nextContext.paragraphId
+  ) {
+    return false
+  }
+
+  const lineEnd = previousLine.endsWith("\r") ? previousLine.length - 1 : previousLine.length
+  const trailing = previousLine.slice(previous.offset + previous.text.length, lineEnd)
+  const leading = nextLine.slice(nextContext.contentStart, token.offset)
+  return (trailing === "" || trailing === " ") && /^[\t ]*$/.test(leading)
+}
+
+const hasWords = (
+  lines: readonly string[],
+  contexts: readonly MarkdownContext[],
+  tokens: readonly WordToken[],
+  start: number,
+  words: readonly string[],
+): boolean =>
+  words.every((word, index) => {
+    const token = tokens[start + index]
+    if (token === undefined || token.lower !== word) {
+      return false
+    }
+    if (index === 0) {
+      return true
+    }
+    const previous = tokens[start + index - 1]
+    if (previous === undefined) {
+      return false
+    }
+    if (token.lineIndex === previous.lineIndex) {
+      const line = lines[token.lineIndex]
+      return (
+        line !== undefined &&
+        /^\s+$/.test(line.slice(previous.offset + previous.text.length, token.offset))
+      )
+    }
+    return isSoftLineBreak(lines, contexts, previous, token)
+  })
+
+const hasPartOfSpeech = (
+  entry: DictionaryEntry,
+  token: WordToken,
+  taggedTokens: readonly TaggedToken[] | undefined,
+): boolean => {
+  if (entry.partsOfSpeech === undefined) {
+    return true
+  }
+  return (
+    taggedTokens?.some(
+      (tagged) =>
+        tagged.offset === token.offset && entry.partsOfSpeech?.includes(tagged.pos) === true,
+    ) === true
+  )
+}
+
+const messageFor = (suggestions: readonly string[], found: string): string => {
+  const alternatives = suggestions.map((suggestion) => `"${suggestion}"`).join(" or ")
+  return `Use ${alternatives}, not "${found}".`
+}
+
+export function dictionaryRule(
+  lines: readonly string[],
+  dictionary: Dictionary,
+  tagger?: Tagger,
+  contentStarts: readonly number[] = lines.map(() => 0),
+): Violation[] {
+  const forms = compileForms(dictionary)
+  const violations: Violation[] = []
+  const contexts = markdownContexts(lines, contentStarts)
+  const tokens = tokenize(lines)
+  const taggedTokensByLine = new Map<number, readonly TaggedToken[]>()
+
+  for (let index = 0; index < tokens.length; index++) {
+    const first = tokens[index]
+    if (first === undefined) {
+      continue
+    }
+    const candidates = forms.filter((form) => hasWords(lines, contexts, tokens, index, form.words))
+    const match = candidates.find((form) => {
+      if (form.entry.partsOfSpeech === undefined) {
+        return true
+      }
+      if (tagger === undefined) {
+        return false
+      }
+      let taggedTokens = taggedTokensByLine.get(first.lineIndex)
+      if (taggedTokens === undefined) {
+        const line = lines[first.lineIndex]
+        if (line === undefined) {
+          return false
+        }
+        taggedTokens = tagger(line)
+        taggedTokensByLine.set(first.lineIndex, taggedTokens)
+      }
+      return hasPartOfSpeech(form.entry, first, taggedTokens)
+    })
+    if (match === undefined) {
+      continue
+    }
+
+    const last = tokens[index + match.words.length - 1]
+    if (last === undefined) {
+      continue
+    }
+    const found =
+      first.lineIndex === last.lineIndex
+        ? lines[first.lineIndex]?.slice(first.offset, last.offset + last.text.length)
+        : tokens
+            .slice(index, index + match.words.length)
+            .map((token) => token.text)
+            .join(" ")
+    if (found === undefined) {
+      continue
+    }
+    violations.push({
+      ruleId: "dictionary-not-approved-word",
+      severity: "hard",
+      message: messageFor(match.entry.suggestions, found),
+      suggestions: match.entry.suggestions,
+      line: first.lineIndex + 1,
+      column: first.offset + 1,
+    })
+    index += match.words.length - 1
+  }
+
+  return violations
+}

package/src/engine/rules/hedging.ts

@@ -0,0 +1,27 @@
+import { scanLines } from "../scan.ts"
+import { TOKEN_CHARACTER_PATTERN } from "../tokens.ts"
+import type { Violation } from "../types.ts"
+
+const HEDGES = [
+  "it is important to note",
+  "it should be noted",
+  "it is worth noting",
+  "please note that",
+  "as mentioned",
+  "as noted above",
+]
+
+const HEDGE_PATTERN = new RegExp(
+  `(?<!${TOKEN_CHARACTER_PATTERN})(?:${HEDGES.map((phrase) => phrase.replace(/ /g, "\\s+")).join("|")})(?!${TOKEN_CHARACTER_PATTERN})`,
+  "gi",
+)
+
+export function hedging(lines: readonly string[]): Violation[] {
+  return scanLines(lines, HEDGE_PATTERN).map((match) => ({
+    ruleId: "hedging",
+    severity: "soft" as const,
+    message: `Do not hedge. Delete "${match.found.toLowerCase()}".`,
+    line: match.line,
+    column: match.column,
+  }))
+}

package/src/engine/rules/marketing.ts

@@ -0,0 +1,71 @@
+import { TOKEN_RUN_PATTERN } from "../tokens.ts"
+import type { Violation } from "../types.ts"
+
+const MARKETING_WORDS = [
+  "seamless",
+  "seamlessly",
+  "robust",
+  "powerful",
+  "cutting-edge",
+  "effortless",
+  "effortlessly",
+  "world-class",
+  "next-generation",
+  "revolutionary",
+  "blazing",
+  "lightning-fast",
+  "elegant",
+  "delightful",
+  "turnkey",
+  "best-in-class",
+  "state-of-the-art",
+  "game-changing",
+  "battle-tested",
+  "enterprise-grade",
+  "supercharge",
+  "unleash",
+  "empower",
+  "empowers",
+]
+
+const MARKETING_SET = new Set(MARKETING_WORDS)
+
+interface MarketingMatch {
+  readonly found: string
+  readonly offset: number
+}
+
+function findMarketingLanguage(token: string): MarketingMatch | undefined {
+  const normalized = token.toLowerCase()
+  if (MARKETING_SET.has(normalized)) {
+    return { found: normalized, offset: 0 }
+  }
+
+  let offset = 0
+  for (const part of normalized.split("-")) {
+    if (MARKETING_SET.has(part)) {
+      return { found: part, offset }
+    }
+    offset += part.length + 1
+  }
+}
+
+export function marketing(lines: readonly string[]): Violation[] {
+  return lines.flatMap((line, lineIndex) =>
+    Array.from(line.matchAll(TOKEN_RUN_PATTERN)).flatMap((tokenMatch) => {
+      const match = findMarketingLanguage(tokenMatch[0])
+      if (!match) {
+        return []
+      }
+      return [
+        {
+          ruleId: "marketing",
+          severity: "soft" as const,
+          message: `Do not use marketing language. Delete "${match.found}".`,
+          line: lineIndex + 1,
+          column: tokenMatch.index + match.offset + 1,
+        },
+      ]
+    }),
+  )
+}

package/src/engine/rules/paragraph-length.ts

@@ -0,0 +1,23 @@
+import type { Paragraph } from "../paragraphs.ts"
+import { segmentSentences } from "../sentences.ts"
+import type { Violation } from "../types.ts"
+
+const MAX_SENTENCES = 6
+
+export function paragraphLength(paragraphs: readonly Paragraph[]): Violation[] {
+  return paragraphs.flatMap((paragraph) => {
+    const count = segmentSentences(paragraph.lines).length
+    if (count <= MAX_SENTENCES) {
+      return []
+    }
+    return [
+      {
+        ruleId: "paragraph-length",
+        severity: "hard" as const,
+        message: `Paragraph has ${count} sentences; the maximum is ${MAX_SENTENCES}.`,
+        line: paragraph.line,
+        column: paragraph.column,
+      },
+    ]
+  })
+}

package/src/engine/rules/phrasal-verb.ts

@@ -0,0 +1,57 @@
+import { scanLines } from "../scan.ts"
+import { TOKEN_CHARACTER_PATTERN } from "../tokens.ts"
+import type { Violation } from "../types.ts"
+
+interface PhrasalVerbEntry {
+  readonly forms: readonly string[]
+  readonly suggestion: string
+}
+
+// List and suggestions follow the pi-ste reference implementation, extended
+// with conjugated forms and "carry out" from the HUF-132 spec.
+const PHRASAL_VERBS: readonly PhrasalVerbEntry[] = [
+  { forms: ["carry out", "carries out", "carried out", "carrying out"], suggestion: "do" },
+  { forms: ["spin up", "spins up", "spun up", "spinning up"], suggestion: "start" },
+  { forms: ["spin down", "spins down", "spun down", "spinning down"], suggestion: "stop" },
+  {
+    forms: ["tear down", "tears down", "tore down", "torn down", "tearing down"],
+    suggestion: "remove",
+  },
+  { forms: ["reach out", "reaches out", "reached out", "reaching out"], suggestion: "ask" },
+  {
+    forms: ["dive into", "dives into", "dived into", "dove into", "diving into"],
+    suggestion: "examine",
+  },
+  { forms: ["kick off", "kicks off", "kicked off", "kicking off"], suggestion: "start" },
+  { forms: ["roll out", "rolls out", "rolled out", "rolling out"], suggestion: "release" },
+  { forms: ["ramp up", "ramps up", "ramped up", "ramping up"], suggestion: "increase" },
+  {
+    forms: ["circle back", "circles back", "circled back", "circling back"],
+    suggestion: "return",
+  },
+  {
+    forms: ["drill down", "drills down", "drilled down", "drilling down"],
+    suggestion: "examine",
+  },
+]
+
+const patterns = PHRASAL_VERBS.map((entry) => ({
+  suggestion: entry.suggestion,
+  pattern: new RegExp(
+    `(?<!${TOKEN_CHARACTER_PATTERN})(?:${entry.forms.map((form) => form.replace(/ /g, "\\s+")).join("|")})(?!${TOKEN_CHARACTER_PATTERN})`,
+    "gi",
+  ),
+}))
+
+export function phrasalVerb(lines: readonly string[]): Violation[] {
+  return patterns.flatMap(({ pattern, suggestion }) =>
+    scanLines(lines, pattern).map((match) => ({
+      ruleId: "phrasal-verb",
+      severity: "hard" as const,
+      message: `Do not use a phrasal verb. Use "${suggestion}", not "${match.found.toLowerCase()}".`,
+      line: match.line,
+      column: match.column,
+      suggestion,
+    })),
+  )
+}

package/src/engine/rules/registry.ts

@@ -0,0 +1,15 @@
+export const ruleIds = [
+  "contraction",
+  "dictionary-not-approved-word",
+  "hedging",
+  "marketing",
+  "paragraph-length",
+  "phrasal-verb",
+  "semicolon",
+  "sentence-length",
+  "verb-progressive",
+  "verb-passive",
+  "verb-perfect",
+] as const
+
+export type RuleId = (typeof ruleIds)[number]

package/src/engine/rules/semicolon.ts

@@ -0,0 +1,14 @@
+import { scanLines } from "../scan.ts"
+import type { Violation } from "../types.ts"
+
+const SEMICOLON = /;/g
+
+export function semicolon(lines: readonly string[]): Violation[] {
+  return scanLines(lines, SEMICOLON).map((match) => ({
+    ruleId: "semicolon",
+    severity: "hard" as const,
+    message: "Do not use a semicolon. Write two sentences.",
+    line: match.line,
+    column: match.column,
+  }))
+}

package/src/engine/rules/sentence-length.ts

@@ -0,0 +1,24 @@
+import type { Sentence } from "../sentences.ts"
+import type { Violation } from "../types.ts"
+
+export function sentenceLength(sentences: readonly Sentence[], maxWords: number): Violation[] {
+  return sentences.flatMap((sentence) => {
+    const count = countWords(sentence.text)
+    if (count <= maxWords) {
+      return []
+    }
+    return [
+      {
+        ruleId: "sentence-length",
+        severity: "hard" as const,
+        message: `Sentence has ${count} words; the maximum is ${maxWords}.`,
+        line: sentence.line,
+        column: sentence.column,
+      },
+    ]
+  })
+}
+
+function countWords(text: string): number {
+  return text.split(/\s+/).filter((word) => word !== "").length
+}

package/src/engine/rules/verb-form.ts

@@ -0,0 +1,76 @@
+import type { TaggedToken, Tagger } from "../tagger.ts"
+import type { Violation } from "../types.ts"
+
+const BE_FORMS = new Set(["am", "is", "are", "was", "were", "be", "been", "being"])
+
+const isBeForm = (token: TaggedToken) => BE_FORMS.has(token.text.toLowerCase())
+
+const isPerfectAuxiliary = (token: TaggedToken) => token.pos === "AUX" && token.lemma === "have"
+
+// Adverbs and "not" may sit between the auxiliary and its verb
+// ("was quickly closed", "were not shown") without breaking the construct.
+const isSkippable = (token: TaggedToken) =>
+  token.pos === "ADV" || token.text.toLowerCase() === "not"
+
+const isProgressiveVerb = (token: TaggedToken) =>
+  token.pos === "VERB" && token.text.toLowerCase().endsWith("ing")
+
+// After a be/have auxiliary, any non-"ing" verb is a past participle in
+// practice, which covers irregular forms (broken, sent, written) without a list.
+const isPastParticiple = (token: TaggedToken) =>
+  token.pos === "VERB" && !token.text.toLowerCase().endsWith("ing")
+
+function nextContentToken(tokens: readonly TaggedToken[], start: number): TaggedToken | undefined {
+  for (let i = start; i < tokens.length; i++) {
+    const token = tokens[i]
+    if (token !== undefined && !isSkippable(token)) {
+      return token
+    }
+  }
+  return undefined
+}
+
+export function verbForm(lines: readonly string[], tag: Tagger): Violation[] {
+  const violations: Violation[] = []
+
+  lines.forEach((line, index) => {
+    if (line.trim() === "") {
+      return
+    }
+    const tokens = tag(line)
+
+    tokens.forEach((token, i) => {
+      const head = nextContentToken(tokens, i + 1)
+      if (head === undefined) {
+        return
+      }
+      const found = line.slice(token.offset, head.offset + head.text.length)
+      const position = { line: index + 1, column: token.offset + 1 }
+
+      if (isBeForm(token) && isProgressiveVerb(head)) {
+        violations.push({
+          ruleId: "verb-progressive",
+          severity: "hard",
+          message: `Use a simple tense. Do not use the progressive. Found: "${found}".`,
+          ...position,
+        })
+      } else if (isBeForm(token) && isPastParticiple(head)) {
+        violations.push({
+          ruleId: "verb-passive",
+          severity: "soft",
+          message: `Use the active voice, unless the actor is unknown. Found: "${found}".`,
+          ...position,
+        })
+      } else if (isPerfectAuxiliary(token) && isPastParticiple(head)) {
+        violations.push({
+          ruleId: "verb-perfect",
+          severity: "hard",
+          message: `Use the simple past. Do not use the perfect tense. Found: "${found}".`,
+          ...position,
+        })
+      }
+    })
+  })
+
+  return violations
+}

package/src/engine/scan.ts

@@ -0,0 +1,15 @@
+export interface LineMatch {
+  readonly found: string
+  readonly line: number
+  readonly column: number
+}
+
+export function scanLines(lines: readonly string[], pattern: RegExp): LineMatch[] {
+  return lines.flatMap((line, index) =>
+    Array.from(line.matchAll(pattern), (match) => ({
+      found: match[0],
+      line: index + 1,
+      column: match.index + 1,
+    })),
+  )
+}